diff --git a/Makefile b/Makefile index 1ccdefbcf..736835a66 100644 --- a/Makefile +++ b/Makefile @@ -154,6 +154,27 @@ test: test-gopkgs verify: verify-godeps verify-fmt verify-generate verify-build verify-docs +# +# install targets +# + +# NRI plugin install directory and default index for static plugin discovery. +NRI_PLUGINS_DIR ?= /opt/nri/plugins +NRI_DEFAULT_IDX ?= 90 + +# install-plugins: install plugin binaries to NRI_PLUGINS_DIR with the +# - prefix required by CRI-O/containerd static plugin discovery. +install-plugins: build-plugins + $(Q)set -e; \ + echo "Installing plugins to $(NRI_PLUGINS_DIR)..."; \ + mkdir -p $(NRI_PLUGINS_DIR); \ + for bin in $(PLUGINS); do \ + name=$${bin#nri-}; \ + dst="$(NRI_PLUGINS_DIR)/$(NRI_DEFAULT_IDX)-$$name"; \ + install -m 755 "$(BIN_PATH)/$$bin" "$$dst"; \ + echo " $$dst"; \ + done + # # build targets # diff --git a/cmd/plugins/resctrl-mon/metrics.go b/cmd/plugins/resctrl-mon/metrics.go new file mode 100644 index 000000000..d919bd7b9 --- /dev/null +++ b/cmd/plugins/resctrl-mon/metrics.go @@ -0,0 +1,130 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "path/filepath" + "strings" + + "github.com/intel/goresctrl/pkg/monitor" + "go.opentelemetry.io/otel/attribute" + otelmetric "go.opentelemetry.io/otel/metric" +) + +// coreAETFiles are counter files that appear under mon_PERF_PKG_* but are +// always exported (not gated by perfCounters.enabled). +var coreAETFiles = map[string]bool{ + "core_energy": true, + "activity": true, +} + +// setupMetrics registers OTel instruments via the goresctrl adapter. +func setupMetrics(mgr *monitor.Manager, cfg telemetryConfig, resctrlRoot string, meter otelmetric.Meter) (*monitor.Registration, error) { + return mgr.RegisterOTelInstruments(meter, + monitor.WithFilter(perfCounterFilter(cfg)), + monitor.WithAttributes(groupAttributesFor(resctrlRoot)), + ) +} + +// perfCounterFilter returns a FilterFunc that implements the perf counter gate. +func perfCounterFilter(cfg telemetryConfig) monitor.FilterFunc { + return func(r monitor.Reading) bool { + // Core AET files (core_energy, activity) are always allowed regardless + // of which domain they appear under. + if coreAETFiles[r.Name] { + return true + } + // Non-PERF_PKG domains (mon_L3_*) are always allowed. + if !strings.HasPrefix(r.Domain, "mon_PERF_PKG_") { + return true + } + // This is a perf counter under mon_PERF_PKG_*. Check the gate. + if !cfg.PerfCounters.Enabled { + return false + } + // Apply include/exclude lists if configured. + if len(cfg.PerfCounters.Include) > 0 { + for _, pattern := range cfg.PerfCounters.Include { + if matchGlob(pattern, r.Name) { + return true + } + } + return false + } + if len(cfg.PerfCounters.Exclude) > 0 { + for _, pattern := range cfg.PerfCounters.Exclude { + if matchGlob(pattern, r.Name) { + return false + } + } + } + return true + } +} + +// groupAttributesFor returns a per-group OTel attribute function bound to the +// configured resctrl root, which is needed to recognize the root ctrl_group. +func groupAttributesFor(resctrlRoot string) monitor.AttributeFunc { + root := filepath.Clean(resctrlRoot) + return func(key, path string) []attribute.KeyValue { + return []attribute.KeyValue{ + attribute.String("k8s.pod.uid", key), + attribute.String("resctrl.control_group", controlGroupOf(root, path)), + // The manager validates and tracks pod UIDs only, so every exported + // group is pod-sourced. + attribute.String("resctrl.group.source", "pod"), + } + } +} + +// matchGlob does simple glob matching (only * is supported as wildcard). +func matchGlob(pattern, name string) bool { + if !strings.Contains(pattern, "*") { + return pattern == name + } + parts := strings.Split(pattern, "*") + // The name must start with the segment before the first '*' and end with + // the segment after the last '*'. + if !strings.HasPrefix(name, parts[0]) { + return false + } + name = name[len(parts[0]):] + last := parts[len(parts)-1] + if !strings.HasSuffix(name, last) { + return false + } + name = name[:len(name)-len(last)] + // Any interior segments must appear in order. + for _, seg := range parts[1 : len(parts)-1] { + i := strings.Index(name, seg) + if i < 0 { + return false + } + name = name[i+len(seg):] + } + return true +} + +// controlGroupOf extracts the CTRL group name from a mon_group path, relative +// to the configured resctrl root. +// e.g. root=/sys/fs/resctrl, "/sys/fs/resctrl/COS1/mon_groups/abc-123" → "COS1" +// e.g. root=/sys/fs/resctrl, "/sys/fs/resctrl/mon_groups/abc-123" → "" (root) +func controlGroupOf(resctrlRoot, groupPath string) string { + ctrlDir := filepath.Dir(filepath.Dir(groupPath)) + if filepath.Clean(ctrlDir) == filepath.Clean(resctrlRoot) { + return "" + } + return filepath.Base(ctrlDir) +} diff --git a/cmd/plugins/resctrl-mon/plugin.go b/cmd/plugins/resctrl-mon/plugin.go index 30f5512a2..36af6db48 100644 --- a/cmd/plugins/resctrl-mon/plugin.go +++ b/cmd/plugins/resctrl-mon/plugin.go @@ -16,6 +16,7 @@ package main import ( "context" + "errors" "fmt" "os" "path/filepath" @@ -25,27 +26,35 @@ import ( "sync" "time" - "github.com/google/uuid" "sigs.k8s.io/yaml" "github.com/containerd/nri/pkg/api" "github.com/containerd/nri/pkg/stub" + "github.com/intel/goresctrl/pkg/monitor" ) const ( // reconcileInterval is how often the background reconciler checks for // orphaned mon_groups left behind by failed StopContainer removals. reconcileInterval = 30 * time.Second + + // telemetryShutdownTimeout bounds how long onClose waits for the telemetry + // stack (MeterProvider flush + HTTP server) to drain before exiting. + telemetryShutdownTimeout = 5 * time.Second ) // plugin implements the NRI plugin interface for resctrl monitoring groups. type plugin struct { stub stub.Stub config *pluginConfig - state *podState - rdt *resctrlOps - mu sync.Mutex // serializes ensureMonGroup to prevent TOCTOU races + mgr *monitor.Manager stopReconciler chan struct{} // closed to stop the background reconciler + telemetry *telemetryState + metrics *monitor.Registration + + mu sync.Mutex // guards pendingRemoval and liveKeys + pendingRemoval map[string]struct{} // keys whose Remove failed, retried by the reconciler + liveKeys map[string]struct{} // monitored pod sandboxes alive as of the last Synchronize } // pluginConfig holds the runtime configuration for the plugin. @@ -60,16 +69,29 @@ type pluginConfig struct { // LabelSelector filters mon_group creation to pods matching these labels. // Empty map means all pods. LabelSelector map[string]string `json:"labelSelector"` + + // Telemetry configures the embedded OTel exporter (Prometheus + OTLP). + Telemetry telemetryConfig `json:"telemetry"` } +const defaultResctrlPath = "/sys/fs/resctrl" + func newPlugin() *plugin { cfg := &pluginConfig{ ResctrlPath: defaultResctrlPath, + Telemetry: defaultTelemetryConfig(), + } + mgr, err := monitor.New(monitor.Options{ + ResctrlRoot: cfg.ResctrlPath, + KeyValidator: monitor.PodUIDValidator, + KeyCanonicalizer: monitor.CanonicalizePodUID, + }) + if err != nil { + log.Fatalf("failed to create monitor manager: %v", err) } return &plugin{ config: cfg, - state: newPodState(), - rdt: newResctrlOps(cfg.ResctrlPath), + mgr: mgr, } } @@ -85,6 +107,15 @@ func (p *plugin) Configure(ctx context.Context, config, runtime, version string) return 0, err } } + // Start telemetry now that configuration (from a --config file and/or the + // NRI server) is finalized. Binding here rather than in main() lets a + // runtime-provided config disable Prometheus or pick a different port + // before we bind, instead of fatally exiting on a pre-config port clash. + if p.telemetry == nil { + if err := p.startTelemetry(ctx); err != nil { + return 0, err + } + } return 0, nil } @@ -93,6 +124,15 @@ func (p *plugin) onClose() { if p.stopReconciler != nil { close(p.stopReconciler) } + if p.metrics != nil { + _ = p.metrics.Unregister() + p.metrics = nil + } + if p.telemetry != nil { + ctx, cancel := context.WithTimeout(context.Background(), telemetryShutdownTimeout) + p.telemetry.shutdown(ctx) + cancel() + } log.Infof("Connection to the runtime lost, exiting...") os.Exit(0) } @@ -102,6 +142,7 @@ func (p *plugin) setConfig(data []byte) error { log.Tracef("setConfig: parsing\n---8<---\n%s\n--->8---", data) cfg := pluginConfig{ ResctrlPath: defaultResctrlPath, + Telemetry: defaultTelemetryConfig(), } if err := yaml.Unmarshal(data, &cfg); err != nil { return fmt.Errorf("setConfig: cannot parse configuration: %w", err) @@ -111,8 +152,102 @@ func (p *plugin) setConfig(data []byte) error { return fmt.Errorf("setConfig: resctrlPath must be an absolute path, got %q", cfg.ResctrlPath) } cfg.ResctrlPath = resctrlPath + if err := validateTelemetryConfig(&cfg.Telemetry); err != nil { + return fmt.Errorf("setConfig: %w", err) + } + + // The resctrl root cannot be changed once the plugin is running: swapping + // the manager would drop the in-memory tracking (and assigned PIDs) for + // every live pod with no re-synchronization until the next lifecycle event, + // and a pending removal bound to the old manager could later delete a + // same-UID group created in the new one. The initial configuration is + // applied (from a --config file and/or the NRI server) before telemetry and + // the reconciler start, so it may still select a non-default root and + // rebuild the manager; only reject the change once the plugin is running. + running := p.telemetry != nil || p.stopReconciler != nil + if running && cfg.ResctrlPath != p.config.ResctrlPath { + return fmt.Errorf("setConfig: resctrlPath cannot be changed on a running plugin (have %q, got %q); restart the plugin to change it", + p.config.ResctrlPath, cfg.ResctrlPath) + } + + // Create the monitor manager on the initial configuration only (the root is + // immutable thereafter, per the guard above). The manager holds the + // in-memory tracking for every running pod. Build it before mutating state so + // a failure here leaves the running configuration fully intact. + rootChanged := p.config == nil || cfg.ResctrlPath != p.config.ResctrlPath + var newMgr *monitor.Manager + if rootChanged { + var err error + newMgr, err = monitor.New(monitor.Options{ + ResctrlRoot: cfg.ResctrlPath, + KeyValidator: monitor.PodUIDValidator, + KeyCanonicalizer: monitor.CanonicalizePodUID, + }) + if err != nil { + return fmt.Errorf("setConfig: failed to create monitor manager: %w", err) + } + } + + // Remember the currently-applied config/manager so a failed reload can be + // rolled back to the last working state instead of leaving telemetry and + // the reconciler permanently disabled. + prevConfig := p.config + prevMgr := p.mgr + p.config = &cfg - p.rdt = newResctrlOps(cfg.ResctrlPath) + + // If telemetry is already running, tear it down so it can be rebound to the + // (possibly new) manager with the new telemetry settings below. + restartTelemetry := p.telemetry != nil + if restartTelemetry { + if p.metrics != nil { + _ = p.metrics.Unregister() + p.metrics = nil + } + ctx, cancel := context.WithTimeout(context.Background(), telemetryShutdownTimeout) + p.telemetry.shutdown(ctx) + cancel() + p.telemetry = nil + } + + // Swap the manager only on a root change. The background reconciler pins the + // manager it was started with, so stop it here and restart it below against + // the new manager; on an unchanged root it keeps running untouched. + reconcilerWasRunning := p.stopReconciler != nil + if rootChanged { + if p.stopReconciler != nil { + close(p.stopReconciler) + p.stopReconciler = nil + } + p.mgr = newMgr + } + + if restartTelemetry { + if err := p.startTelemetry(context.Background()); err != nil { + // Roll back to the previously working configuration: the old + // telemetry/manager have already been torn down, so restore the + // prior config/manager and bring telemetry (and, if we swapped the + // manager, the reconciler) back up. This keeps a failed reload + // (e.g. the new Prometheus port is occupied) from permanently + // disabling telemetry and reconciliation. + p.config = prevConfig + if rootChanged { + p.mgr = prevMgr + } + if rerr := p.startTelemetry(context.Background()); rerr != nil { + log.Errorf("setConfig: failed to restore telemetry after failed reload: %v", rerr) + } + if rootChanged && reconcilerWasRunning { + p.startReconciler() + } + return fmt.Errorf("setConfig: restart telemetry: %w", err) + } + } + + if rootChanged && reconcilerWasRunning { + p.startReconciler() + } + log.Debugf("configuration: resctrlPath=%s namespaces=%v labelSelector=%v", cfg.ResctrlPath, cfg.Namespaces, cfg.LabelSelector) return nil @@ -130,8 +265,19 @@ func (p *plugin) Synchronize(ctx context.Context, pods []*api.PodSandbox, contai podBySandboxID[pod.GetId()] = pod } - // Create mon_groups for running containers that don't have one, - // and write their PIDs to ensure monitoring is active after restart. + // A pod sandbox can be alive with no running container (for example between + // container restarts). Seed the reconcile live set from the monitored + // sandboxes so their existing mon_groups survive; the container loop below + // then creates missing groups and (re)assigns PIDs. Remember this set so the + // background reconciler keeps protecting container-less sandboxes (which are + // never passed to EnsureGroup and so never appear in mgr.List()). + liveKeys := make([]string, 0, len(pods)) + for _, pod := range pods { + if p.shouldMonitorPod(pod) { + liveKeys = append(liveKeys, pod.GetUid()) + } + } + p.setLiveKeys(liveKeys) for _, ctr := range containers { pod, ok := podBySandboxID[ctr.GetPodSandboxId()] if !ok { @@ -143,59 +289,155 @@ func (p *plugin) Synchronize(ctx context.Context, pods []*api.PodSandbox, contai } podUID := pod.GetUid() rdtClass := getRDTClass(ctr) - if err := p.ensureMonGroup(podUID, ctr.GetId(), rdtClass); err != nil { + + grp, err := p.mgr.EnsureGroup(podUID, rdtClass) + if err != nil { log.Warnf("Synchronize: failed to create mon_group for pod %s: %v", podUID, err) continue } - // Use canonical form for state lookups (ensureMonGroup stores canonical). - u, _ := uuid.Parse(podUID) - canonicalUID := u.String() + pid := int(ctr.GetPid()) if pid > 0 { - monGroupDir := p.state.getMonGroupDir(canonicalUID) - if err := p.rdt.writeTaskPID(monGroupDir, pid); err != nil { + if err := p.mgr.AssignPID(podUID, pid); err != nil { log.Warnf("Synchronize: failed to write PID %d for pod %s: %v", pid, podUID, err) } else { - log.Debugf("Synchronize: assigned pid %d for pod %s", pid, podUID) + log.Debugf("Synchronize: assigned pid %d for pod %s in %s", pid, podUID, grp.Path()) } } } // Remove orphaned mon_groups from a previous plugin instance. - p.rdt.cleanOrphanedMonGroups(p.state) + if err := p.mgr.Reconcile(liveKeys); err != nil { + log.Warnf("Synchronize: reconcile failed: %v", err) + } // Start the background reconciler to periodically clean up orphaned // mon_groups that could not be removed during StopContainer. p.startReconciler() - log.Infof("synchronization complete: tracking %d pods", p.state.podCount()) + log.Infof("synchronization complete: tracking %d pods", len(p.mgr.List())) return nil, nil } -// startReconciler launches a background goroutine that periodically removes -// orphaned mon_group directories. This handles the case where removeMonGroup -// fails in StopContainer (e.g., kernel busy) and the directory lingers. +// startReconciler launches a background goroutine that periodically retries +// failed removals and removes orphaned mon_group directories. This handles the +// case where removeMonGroup fails in RemovePodSandbox (e.g., kernel busy) and +// the directory lingers. func (p *plugin) startReconciler() { if p.stopReconciler != nil { // Already running from a previous Synchronize call. return } - p.stopReconciler = make(chan struct{}) + // Capture the channel and manager this goroutine owns. A later config reload + // may close p.stopReconciler, swap p.mgr, and start a fresh goroutine; binding + // to these locals keeps this goroutine's select on an immutable channel and + // pinned to the manager it was started with. + stop := make(chan struct{}) + p.stopReconciler = stop + mgr := p.mgr go func() { ticker := time.NewTicker(reconcileInterval) defer ticker.Stop() for { select { - case <-p.stopReconciler: + case <-stop: return case <-ticker.C: - p.rdt.cleanOrphanedMonGroups(p.state) + p.reconcile(mgr) } } }() log.Debugf("background reconciler started (interval=%s)", reconcileInterval) } +// reconcile retries removals that previously failed and reaps untracked orphan +// directories. A failed Remove leaves its key tracked, so Reconcile(List()) +// would treat it as live forever; retrying Remove is what actually frees the +// RMID once the kernel releases the directory. +func (p *plugin) reconcile(mgr *monitor.Manager) { + for _, key := range p.pendingRemovalKeys() { + switch err := mgr.Remove(key); { + case err == nil, errors.Is(err, monitor.ErrNotTracked): + p.clearPendingRemoval(key) + default: + log.Warnf("reconciler: retry remove %s failed: %v", key, err) + } + } + // Reconcile against the tracked keys plus the live sandbox set: a + // container-less sandbox is protected only by liveKeys (it is never passed + // to EnsureGroup, so mgr.List() omits it), and reconciling without it would + // reap its existing mon_group and hand its replacement container a fresh + // RMID. + if err := mgr.Reconcile(p.reconcileLiveSet(mgr)); err != nil { + log.Warnf("reconciler: %v", err) + } +} + +// setLiveKeys records the set of monitored pod sandboxes that are alive, so the +// background reconciler can protect their mon_groups even when no container has +// caused them to be tracked in the Manager. +func (p *plugin) setLiveKeys(keys []string) { + p.mu.Lock() + defer p.mu.Unlock() + p.liveKeys = make(map[string]struct{}, len(keys)) + for _, k := range keys { + p.liveKeys[k] = struct{}{} + } +} + +// dropLiveKey removes a sandbox key from the live set once its pod is gone, so +// the reconciler stops protecting it and can reap any leftover mon_group. +func (p *plugin) dropLiveKey(key string) { + p.mu.Lock() + defer p.mu.Unlock() + delete(p.liveKeys, key) +} + +// reconcileLiveSet returns the union of the Manager's tracked keys and the live +// sandbox set for use as the reconcile live list. +func (p *plugin) reconcileLiveSet(mgr *monitor.Manager) []string { + p.mu.Lock() + defer p.mu.Unlock() + live := make(map[string]struct{}, len(p.liveKeys)) + for k := range p.liveKeys { + live[k] = struct{}{} + } + for _, k := range mgr.List() { + live[k] = struct{}{} + } + keys := make([]string, 0, len(live)) + for k := range live { + keys = append(keys, k) + } + return keys +} + +// markPendingRemoval records a key whose Remove failed so the reconciler retries it. +func (p *plugin) markPendingRemoval(key string) { + p.mu.Lock() + defer p.mu.Unlock() + if p.pendingRemoval == nil { + p.pendingRemoval = make(map[string]struct{}) + } + p.pendingRemoval[key] = struct{}{} +} + +func (p *plugin) clearPendingRemoval(key string) { + p.mu.Lock() + defer p.mu.Unlock() + delete(p.pendingRemoval, key) +} + +func (p *plugin) pendingRemovalKeys() []string { + p.mu.Lock() + defer p.mu.Unlock() + keys := make([]string, 0, len(p.pendingRemoval)) + for k := range p.pendingRemoval { + keys = append(keys, k) + } + return keys +} + // PostCreateContainer is called after the container is created but before // it starts executing. The container PID is NOT yet available (pid=0) because // the init process has not been started. We create the mon_group here so it @@ -212,7 +454,7 @@ func (p *plugin) PostCreateContainer(ctx context.Context, pod *api.PodSandbox, c } rdtClass := getRDTClass(ctr) - if err := p.ensureMonGroup(podUID, ctr.GetId(), rdtClass); err != nil { + if _, err := p.mgr.EnsureGroup(podUID, rdtClass); err != nil { log.Warnf("PostCreateContainer %s: failed to create mon_group: %v", ctrName, err) return nil // non-fatal: don't block container creation } @@ -227,14 +469,8 @@ func (p *plugin) PostCreateContainer(ctx context.Context, pod *api.PodSandbox, c // This is the ideal moment to write the PID to the resctrl mon_group tasks // file: the kernel assigns the RMID to this PID, and when the process starts // and forks threads they all inherit the RMID automatically. -// -// If the PID is not available (should not happen at this stage), we fall back -// to PostStartContainer which will write PIDs after the process starts. func (p *plugin) StartContainer(ctx context.Context, pod *api.PodSandbox, ctr *api.Container) error { podUID := pod.GetUid() - if u, err := uuid.Parse(podUID); err == nil { - podUID = u.String() - } ctrName := pprintCtr(pod, ctr) pid := int(ctr.GetPid()) @@ -244,17 +480,11 @@ func (p *plugin) StartContainer(ctx context.Context, pod *api.PodSandbox, ctr *a return nil } - monGroupDir := p.state.getMonGroupDir(podUID) - if monGroupDir == "" { - log.Debugf("StartContainer %s: no mon_group (pod not tracked), skipping", ctrName) - return nil - } - if pid > 0 { - if err := p.rdt.writeTaskPID(monGroupDir, pid); err != nil { - log.Warnf("StartContainer %s: failed to write PID %d to tasks: %v", ctrName, pid, err) + if err := p.mgr.AssignPID(podUID, pid); err != nil { + log.Warnf("StartContainer %s: failed to assign PID %d: %v", ctrName, pid, err) } else { - log.Infof("StartContainer %s: assigned pid %d to mon_group %s (pre-start, no threads yet)", ctrName, pid, monGroupDir) + log.Infof("StartContainer %s: assigned pid %d (pre-start, no threads yet)", ctrName, pid) } } else { log.Warnf("StartContainer %s: PID not available at pre-start, will retry in PostStartContainer", ctrName) @@ -265,13 +495,9 @@ func (p *plugin) StartContainer(ctx context.Context, pod *api.PodSandbox, ctr *a // PostStartContainer is called after the container process has been started. // This is a fallback: if StartContainer did not have the PID, we write the -// init PID here. The init PID is sufficient because all child threads inherit -// the RMID. +// init PID here. func (p *plugin) PostStartContainer(ctx context.Context, pod *api.PodSandbox, ctr *api.Container) error { podUID := pod.GetUid() - if u, err := uuid.Parse(podUID); err == nil { - podUID = u.String() - } ctrName := pprintCtr(pod, ctr) pid := int(ctr.GetPid()) @@ -281,16 +507,11 @@ func (p *plugin) PostStartContainer(ctx context.Context, pod *api.PodSandbox, ct return nil } - monGroupDir := p.state.getMonGroupDir(podUID) - if monGroupDir == "" { - return nil - } - if pid > 0 { - if err := p.rdt.writeTaskPID(monGroupDir, pid); err != nil { - log.Warnf("PostStartContainer %s: failed to write PID %d to tasks: %v", ctrName, pid, err) + if err := p.mgr.AssignPID(podUID, pid); err != nil { + log.Warnf("PostStartContainer %s: failed to assign PID %d: %v", ctrName, pid, err) } else { - log.Infof("PostStartContainer %s: assigned pid %d to mon_group %s", ctrName, pid, monGroupDir) + log.Infof("PostStartContainer %s: assigned pid %d", ctrName, pid) } } else { log.Warnf("PostStartContainer %s: PID=0, cannot assign to mon_group (runtime did not provide PID via NRI)", ctrName) @@ -299,97 +520,42 @@ func (p *plugin) PostStartContainer(ctx context.Context, pod *api.PodSandbox, ct return nil } -// StopContainer is called when a container is being stopped. -func (p *plugin) StopContainer(ctx context.Context, pod *api.PodSandbox, ctr *api.Container) ([]*api.ContainerUpdate, error) { - podUID := pod.GetUid() - if u, err := uuid.Parse(podUID); err == nil { - podUID = u.String() - } - ctrName := pprintCtr(pod, ctr) - - log.Debugf("StopContainer %s", ctrName) - - monGroupDir := p.state.getMonGroupDir(podUID) - if monGroupDir == "" { - return nil, nil - } - - // Drop only the container from tracking. The mon_group is intentionally - // retained until the pod sandbox is removed (see RemovePodSandbox). - // - // A pod with restartPolicy Always/OnFailure restarts its container under - // the same pod UID after the process exits (e.g. a workload that runs for - // a fixed duration). If we removed the mon_group here, the kernel would - // release the RMID and reassign a fresh one on the next PostCreateContainer. - // The new RMID carries residual hardware counter values, producing a - // counter discontinuity that surfaces as a false energy/bandwidth spike in - // downstream rate() consumers. Tying mon_group lifetime to the pod sandbox - // keeps the RMID stable across container restarts. - p.state.removeContainer(podUID, ctr.GetId()) - - if p.state.podHasNoContainers(podUID) { - log.Debugf("StopContainer %s: last container stopped, retaining mon_group %s until pod removal", ctrName, monGroupDir) - } - - return nil, nil -} - -// RemovePodSandbox is called when a pod sandbox is torn down. This is the point -// at which the pod (and its UID) is truly gone, so it is the correct place to -// release the mon_group and its RMID. Removing the mon_group earlier (e.g. in -// StopContainer) would release the RMID across container restarts and cause -// false counter spikes; see StopContainer for details. +// StopContainer is intentionally not implemented. A container stop must NOT +// tear down the pod's mon_group: a restart keeps the pod sandbox alive, and +// releasing the RMID would give the replacement container a fresh RMID whose +// hardware counters carry a non-zeroed residual, producing a false energy +// spike. The mon_group is removed in RemovePodSandbox when the pod is truly +// gone (and the reconciler cleans orphans from any missed teardown events). +// Because the NRI stub derives its event subscription from the implemented +// handler interfaces, omitting StopContainer also unsubscribes the plugin from +// STOP_CONTAINER events entirely. + +// RemovePodSandbox is called when the pod sandbox is being torn down. +// This is the point at which the mon_group should be cleaned up, because +// the pod (and its UID) will not be reused. func (p *plugin) RemovePodSandbox(ctx context.Context, pod *api.PodSandbox) error { podUID := pod.GetUid() - if u, err := uuid.Parse(podUID); err == nil { - podUID = u.String() - } - monGroupDir := p.state.getMonGroupDir(podUID) - if monGroupDir == "" { - return nil - } - - log.Infof("RemovePodSandbox %s/%s: removing mon_group %s", pod.GetNamespace(), pod.GetName(), monGroupDir) - if err := p.rdt.removeMonGroup(monGroupDir); err != nil { - log.Warnf("RemovePodSandbox %s/%s: failed to remove mon_group (will be cleaned by reconciler): %v", + // The sandbox is gone, so stop protecting its key in the reconciler's live + // set; otherwise a failed Remove below could never be reaped. + p.dropLiveKey(podUID) + + // Attempt removal unconditionally rather than gating on shouldMonitorPod: a + // pod may have been monitored under a configuration that was later changed to + // exclude it. Gating here would strand its mon_group, because Remove would + // never run and the key would linger in the Manager, so the reconciler would + // keep treating it as live and never reap it. Remove is idempotent and + // reports ErrNotTracked for a pod that was never monitored. + switch err := p.mgr.Remove(podUID); { + case err == nil: + log.Infof("RemovePodSandbox %s/%s: removed mon_group", pod.GetNamespace(), pod.GetName()) + case errors.Is(err, monitor.ErrNotTracked): + // Pod was never monitored; nothing to clean up. + default: + log.Warnf("RemovePodSandbox %s/%s: failed to remove mon_group (will be retried by reconciler): %v", pod.GetNamespace(), pod.GetName(), err) + p.markPendingRemoval(podUID) } - p.state.removePod(podUID) - - return nil -} - -// ensureMonGroup creates the mon_group directory if it doesn't exist and registers -// the container in the in-memory state. -// -// Limitation: all containers in a pod share a single mon_group under the first -// container's RDT class. If an allocation plugin assigns different classes to -// containers in the same pod, subsequent containers use the first class. -func (p *plugin) ensureMonGroup(podUID, containerID, rdtClass string) error { - u, err := uuid.Parse(podUID) - if err != nil { - return fmt.Errorf("invalid pod UID %q", podUID) - } - podUID = u.String() - - p.mu.Lock() - defer p.mu.Unlock() - - if p.state.getMonGroupDir(podUID) != "" { - // Mon_group already exists for this pod. Just add the container. - p.state.addContainer(podUID, containerID) - return nil - } - - monGroupDir, err := p.rdt.createMonGroup(rdtClass, podUID) - if err != nil { - return err - } - - p.state.addPod(podUID, monGroupDir) - p.state.addContainer(podUID, containerID) - log.Infof("created mon_group %s for pod %s", monGroupDir, podUID) return nil } diff --git a/cmd/plugins/resctrl-mon/plugin_test.go b/cmd/plugins/resctrl-mon/plugin_test.go index a5d7215f1..60507f894 100644 --- a/cmd/plugins/resctrl-mon/plugin_test.go +++ b/cmd/plugins/resctrl-mon/plugin_test.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/containerd/nri/pkg/api" + "github.com/intel/goresctrl/pkg/monitor" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -35,10 +36,17 @@ func newTestPlugin(resctrlPath string) *plugin { cfg := &pluginConfig{ ResctrlPath: resctrlPath, } + mgr, err := monitor.New(monitor.Options{ + ResctrlRoot: resctrlPath, + KeyValidator: monitor.PodUIDValidator, + KeyCanonicalizer: monitor.CanonicalizePodUID, + }) + if err != nil { + panic(err) + } return &plugin{ config: cfg, - state: newPodState(), - rdt: newResctrlOps(resctrlPath), + mgr: mgr, } } @@ -129,23 +137,27 @@ func TestPostCreateContainer_FilteredPod(t *testing.T) { require.NoError(t, err) // Pod should not be tracked since it's not in the production namespace. - assert.Equal(t, 0, p.state.podCount()) + assert.Equal(t, 0, len(p.mgr.List())) } func TestPostCreateContainer_CreatesMonGroup(t *testing.T) { tmpDir := t.TempDir() p := newTestPlugin(tmpDir) - pod := makePod("a1b2c3d4-e5f6-7890-abcd-ef1234567890", "default", "test-pod") - ctr := makeContainer("c1", "container1", "a1b2c3d4-e5f6-7890-abcd-ef1234567890", 0, "") + podUID := "a1b2c3d4-e5f6-7890-abcd-ef1234567890" + pod := makePod(podUID, "default", "test-pod") + ctr := makeContainer("c1", "container1", podUID, 0, "") err := p.PostCreateContainer(context.Background(), pod, ctr) require.NoError(t, err) // Pod should be tracked. - assert.Equal(t, 1, p.state.podCount()) - monDir := p.state.getMonGroupDir("a1b2c3d4-e5f6-7890-abcd-ef1234567890") - assert.Contains(t, monDir, "mon_groups/a1b2c3d4-e5f6-7890-abcd-ef1234567890") + assert.Equal(t, 1, len(p.mgr.List())) + + // Mon_group directory should exist, keyed by bare pod UID. + monDir := filepath.Join(tmpDir, "mon_groups", podUID) + _, err = os.Stat(monDir) + assert.NoError(t, err) } func TestPostCreateContainer_WithRDTClass(t *testing.T) { @@ -153,14 +165,17 @@ func TestPostCreateContainer_WithRDTClass(t *testing.T) { p := newTestPlugin(tmpDir) require.NoError(t, os.Mkdir(filepath.Join(tmpDir, "BestEffort"), 0755)) - pod := makePod("a1b2c3d4-e5f6-7890-abcd-ef1234567890", "default", "test-pod") - ctr := makeContainer("c1", "container1", "a1b2c3d4-e5f6-7890-abcd-ef1234567890", 0, "BestEffort") + podUID := "a1b2c3d4-e5f6-7890-abcd-ef1234567890" + pod := makePod(podUID, "default", "test-pod") + ctr := makeContainer("c1", "container1", podUID, 0, "BestEffort") err := p.PostCreateContainer(context.Background(), pod, ctr) require.NoError(t, err) - monDir := p.state.getMonGroupDir("a1b2c3d4-e5f6-7890-abcd-ef1234567890") - assert.Contains(t, monDir, "BestEffort/mon_groups/a1b2c3d4-e5f6-7890-abcd-ef1234567890") + // Mon_group should be under the ctrl_group. + monDir := filepath.Join(tmpDir, "BestEffort", "mon_groups", podUID) + _, err = os.Stat(monDir) + assert.NoError(t, err) } func TestMultiContainerPod(t *testing.T) { @@ -175,127 +190,25 @@ func TestMultiContainerPod(t *testing.T) { // First container creates the mon_group. err := p.PostCreateContainer(context.Background(), pod, ctr1) require.NoError(t, err) - assert.Equal(t, 1, p.state.podCount()) + assert.Equal(t, 1, len(p.mgr.List())) // Second container reuses the same mon_group. err = p.PostCreateContainer(context.Background(), pod, ctr2) require.NoError(t, err) - assert.Equal(t, 1, p.state.podCount()) // still one pod - - // Stopping first container should not remove the mon_group. - _, err = p.StopContainer(context.Background(), pod, ctr1) - require.NoError(t, err) - assert.Equal(t, 1, p.state.podCount()) - assert.False(t, p.state.podHasNoContainers(podUID)) - - // Stopping the last container retains the mon_group (it is released only - // when the pod sandbox is removed, so the RMID stays stable across - // container restarts). - _, err = p.StopContainer(context.Background(), pod, ctr2) - require.NoError(t, err) - assert.Equal(t, 1, p.state.podCount()) - assert.True(t, p.state.podHasNoContainers(podUID)) - - // Removing the pod sandbox releases the mon_group. - err = p.RemovePodSandbox(context.Background(), pod) - require.NoError(t, err) - assert.Equal(t, 0, p.state.podCount()) -} - -func TestStopContainer_UnknownPod(t *testing.T) { - p := newTestPlugin(t.TempDir()) - - pod := makePod("unknown-uid", "default", "unknown-pod") - ctr := makeContainer("c1", "container1", "unknown-uid", 1234, "") - - updates, err := p.StopContainer(context.Background(), pod, ctr) - require.NoError(t, err) - assert.Nil(t, updates) -} - -// TestContainerRestart_RetainsMonGroup verifies that a container restart under -// the same pod UID (e.g. restartPolicy: Always after a fixed-duration workload -// exits) keeps the same mon_group directory, so the kernel does not release and -// reassign the RMID. RMID reassignment would carry residual hardware counter -// values and surface as a false counter spike. -func TestContainerRestart_RetainsMonGroup(t *testing.T) { - tmpDir := t.TempDir() - p := newTestPlugin(tmpDir) - podUID := "a1b2c3d4-e5f6-7890-abcd-ef1234567890" - - pod := makePod(podUID, "default", "restart-pod") - ctr := makeContainer("c1", "container1", podUID, 0, "") - - // Container starts: mon_group is created. - err := p.PostCreateContainer(context.Background(), pod, ctr) - require.NoError(t, err) - monDir := p.state.getMonGroupDir(podUID) - require.NotEmpty(t, monDir) - require.DirExists(t, monDir) - - // Container exits (workload timed out). The mon_group must be retained. - _, err = p.StopContainer(context.Background(), pod, ctr) - require.NoError(t, err) - assert.Equal(t, 1, p.state.podCount()) - assert.DirExists(t, monDir, "mon_group must survive a container restart") - - // kubelet restarts the container under the same pod UID. The same - // mon_group directory (and thus RMID) must be reused. - err = p.PostCreateContainer(context.Background(), pod, ctr) - require.NoError(t, err) - assert.Equal(t, monDir, p.state.getMonGroupDir(podUID), "restart must reuse the same mon_group") + assert.Equal(t, 1, len(p.mgr.List())) // still one pod - // Pod is finally deleted: mon_group is released. + // RemovePodSandbox is what actually removes the mon_group; container + // stops do not affect it (the plugin no longer handles StopContainer). err = p.RemovePodSandbox(context.Background(), pod) require.NoError(t, err) - assert.Equal(t, 0, p.state.podCount()) - assert.NoDirExists(t, monDir) -} - -// TestRemovePodSandbox_UnknownPod verifies that removing a pod we never tracked -// is a no-op and does not error. -func TestRemovePodSandbox_UnknownPod(t *testing.T) { - p := newTestPlugin(t.TempDir()) - - pod := makePod("a1b2c3d4-e5f6-7890-abcd-ef1234567890", "default", "unknown-pod") - - err := p.RemovePodSandbox(context.Background(), pod) - require.NoError(t, err) - assert.Equal(t, 0, p.state.podCount()) -} - -// TestMissingPodUID_NoMonGroup verifies that a sandbox without a valid pod UID -// is handled as a safe no-op across the full lifecycle: no mon_group is created -// (mon_groups are keyed by pod UID, not per container), and the stop/remove -// handlers neither panic nor leak state. This documents that the plugin does -// not currently fall back to per-container monitoring when the UID is absent. -func TestMissingPodUID_NoMonGroup(t *testing.T) { - p := newTestPlugin(t.TempDir()) - - pod := makePod("", "default", "no-uid-pod") - ctr := makeContainer("c1", "container1", "", 1234, "") - - // Creation must not create a group and must be non-fatal. - require.NoError(t, p.PostCreateContainer(context.Background(), pod, ctr)) - assert.Equal(t, 0, p.state.podCount()) - - // The remaining handlers must be safe no-ops. - require.NoError(t, p.StartContainer(context.Background(), pod, ctr)) - require.NoError(t, p.PostStartContainer(context.Background(), pod, ctr)) - - _, err := p.StopContainer(context.Background(), pod, ctr) - require.NoError(t, err) - assert.Equal(t, 0, p.state.podCount()) - - require.NoError(t, p.RemovePodSandbox(context.Background(), pod)) - assert.Equal(t, 0, p.state.podCount()) + assert.Equal(t, 0, len(p.mgr.List())) } func TestSetConfig(t *testing.T) { p := newTestPlugin("/tmp/resctrl-test") configYAML := []byte(` -resctrlPath: /tmp/test-resctrl +resctrlPath: /tmp/resctrl-test namespaces: - production - staging @@ -305,7 +218,7 @@ labelSelector: err := p.setConfig(configYAML) require.NoError(t, err) - assert.Equal(t, "/tmp/test-resctrl", p.config.ResctrlPath) + assert.Equal(t, "/tmp/resctrl-test", p.config.ResctrlPath) assert.Equal(t, []string{"production", "staging"}, p.config.Namespaces) assert.Equal(t, map[string]string{"monitor": "true"}, p.config.LabelSelector) } @@ -338,25 +251,112 @@ func TestSynchronize_UsesUIDNotSandboxID(t *testing.T) { require.NoError(t, err) // The mon_group should be keyed by the K8s pod UID, not the sandbox ID. - assert.Equal(t, 1, p.state.podCount()) - assert.True(t, p.state.hasPod(podUID)) - assert.False(t, p.state.hasPod(pod.GetId())) + tracked := p.mgr.List() + assert.Equal(t, 1, len(tracked)) + assert.Contains(t, tracked, podUID) + + // Mon_group directory should exist. + monDir := filepath.Join(tmpDir, "mon_groups", podUID) + _, err = os.Stat(monDir) + assert.NoError(t, err) +} + +func TestSynchronize_RemovesOrphanMonGroup(t *testing.T) { + tmpDir := t.TempDir() - monDir := p.state.getMonGroupDir(podUID) - assert.Contains(t, monDir, podUID) + // An orphaned mon_group left behind by a previous run, keyed by a + // UUID-shaped pod UID that is no longer live. + orphanUID := "deadbeef-0000-4000-8000-000000000000" + orphanDir := filepath.Join(tmpDir, "mon_groups", orphanUID) + require.NoError(t, os.MkdirAll(orphanDir, 0755)) + + p := newTestPlugin(tmpDir) + + // Synchronize with a single live pod that is not the orphan. + podUID := "a1b2c3d4-e5f6-7890-abcd-ef1234567890" + pod := makePod(podUID, "default", "live-pod") + ctr := makeContainer("c1", "container1", pod.GetId(), 0, "") + + _, err := p.Synchronize(context.Background(), []*api.PodSandbox{pod}, []*api.Container{ctr}) + require.NoError(t, err) + + // The live pod's mon_group exists... + _, err = os.Stat(filepath.Join(tmpDir, "mon_groups", podUID)) + assert.NoError(t, err) + + // ...and the orphan was reaped via Reconcile. + _, err = os.Stat(orphanDir) + assert.True(t, os.IsNotExist(err), "orphan mon_group should have been removed by Reconcile") } -func TestEnsureMonGroup_InvalidUID(t *testing.T) { - p := newTestPlugin(t.TempDir()) +// TestReconcile_PreservesContainerlessLiveSandbox verifies that a monitored pod +// sandbox that is alive with no running container (its mon_group survives from +// a previous run but no EnsureGroup tracks it) is not reaped by the background +// reconciler, so a restarting container reuses the same RMID. +func TestReconcile_PreservesContainerlessLiveSandbox(t *testing.T) { + tmpDir := t.TempDir() - err := p.ensureMonGroup("", "c1", "") - assert.Error(t, err) - assert.Contains(t, err.Error(), "invalid pod UID") + // A container-less sandbox: its mon_group exists on disk but the plugin + // never calls EnsureGroup for it, so mgr.List() will not include it. + sandboxUID := "a1b2c3d4-e5f6-7890-abcd-ef1234567890" + sandboxDir := filepath.Join(tmpDir, "mon_groups", sandboxUID) + require.NoError(t, os.MkdirAll(sandboxDir, 0755)) - err = p.ensureMonGroup("not-a-uuid", "c1", "") - assert.Error(t, err) + p := newTestPlugin(tmpDir) + + // Synchronize with the live sandbox but no containers. + pod := makePod(sandboxUID, "default", "live-pod") + _, err := p.Synchronize(context.Background(), []*api.PodSandbox{pod}, nil) + require.NoError(t, err) + + // The initial Reconcile(liveKeys) must have kept it. + require.DirExists(t, sandboxDir) + require.NotContains(t, p.mgr.List(), sandboxUID, "container-less sandbox is not tracked in the Manager") - assert.Equal(t, 0, p.state.podCount()) + // A background reconcile tick must still preserve it (regression: it used to + // reconcile against mgr.List() only and reap the group). + p.reconcile(p.mgr) + assert.DirExists(t, sandboxDir) + + // Once the sandbox is removed, it stops being protected and is reaped. + require.NoError(t, p.RemovePodSandbox(context.Background(), pod)) + p.reconcile(p.mgr) + _, err = os.Stat(sandboxDir) + assert.True(t, os.IsNotExist(err), "sandbox mon_group should be reaped after the pod is gone") +} + +func TestReconcile_RetriesPendingRemoval(t *testing.T) { + tmpDir := t.TempDir() + p := newTestPlugin(tmpDir) + podUID := "a1b2c3d4-e5f6-7890-abcd-ef1234567890" + + // A tracked group whose earlier Remove is assumed to have failed. + _, err := p.mgr.EnsureGroup(podUID, "") + require.NoError(t, err) + require.Contains(t, p.mgr.List(), podUID) + p.markPendingRemoval(podUID) + + // The reconciler must retry Remove (not merely Reconcile, which preserves + // tracked keys) and clear the pending entry on success. + p.reconcile(p.mgr) + + assert.NotContains(t, p.mgr.List(), podUID) + assert.Empty(t, p.pendingRemovalKeys()) + _, err = os.Stat(filepath.Join(tmpDir, "mon_groups", podUID)) + assert.True(t, os.IsNotExist(err), "pending mon_group should have been removed on retry") +} + +func TestPostCreateContainer_InvalidUID(t *testing.T) { + p := newTestPlugin(t.TempDir()) + + // Invalid UID (not a UUID) — EnsureGroup fails due to PodUIDValidator. + pod := makePod("not-a-uuid", "default", "bad-pod") + ctr := makeContainer("c1", "container1", "not-a-uuid", 0, "") + + err := p.PostCreateContainer(context.Background(), pod, ctr) + // Non-fatal: returns nil but does not track. + require.NoError(t, err) + assert.Equal(t, 0, len(p.mgr.List())) } func TestStartContainer_AssignsPID(t *testing.T) { @@ -371,8 +371,8 @@ func TestStartContainer_AssignsPID(t *testing.T) { err := p.PostCreateContainer(context.Background(), pod, ctr) require.NoError(t, err) - monDir := p.state.getMonGroupDir(podUID) - require.NotEmpty(t, monDir) + monDir := filepath.Join(tmpDir, "mon_groups", podUID) + require.DirExists(t, monDir) // Simulate the kernel creating the tasks file. require.NoError(t, os.WriteFile(filepath.Join(monDir, "tasks"), nil, 0644)) @@ -399,8 +399,7 @@ func TestStartContainer_PIDZero_FallbackToPostStart(t *testing.T) { err := p.PostCreateContainer(context.Background(), pod, ctr) require.NoError(t, err) - monDir := p.state.getMonGroupDir(podUID) - require.NotEmpty(t, monDir) + monDir := filepath.Join(tmpDir, "mon_groups", podUID) require.NoError(t, os.WriteFile(filepath.Join(monDir, "tasks"), nil, 0644)) // StartContainer with PID 0 should not fail (just warns). @@ -429,82 +428,7 @@ func TestStartContainer_FilteredPod(t *testing.T) { require.NoError(t, err) } -func TestCompactUID_EnsureMonGroupStoresCanonical(t *testing.T) { - tmpDir := t.TempDir() - p := newTestPlugin(tmpDir) - - compactUID := "a1b2c3d4e5f678901234abcdef567890" - canonicalUID := "a1b2c3d4-e5f6-7890-1234-abcdef567890" - - pod := makePod(compactUID, "default", "test-pod") - ctr := makeContainer("c1", "container1", compactUID, 0, "") - - err := p.PostCreateContainer(context.Background(), pod, ctr) - require.NoError(t, err) - - // State must be keyed under the canonical dashed form. - assert.True(t, p.state.hasPod(canonicalUID)) - assert.False(t, p.state.hasPod(compactUID)) - - monDir := p.state.getMonGroupDir(canonicalUID) - assert.Contains(t, monDir, canonicalUID) -} - -func TestCompactUID_StartContainerFindsMonGroup(t *testing.T) { - tmpDir := t.TempDir() - p := newTestPlugin(tmpDir) - - compactUID := "a1b2c3d4e5f678901234abcdef567890" - canonicalUID := "a1b2c3d4-e5f6-7890-1234-abcdef567890" - - pod := makePod(compactUID, "default", "test-pod") - ctr := makeContainer("c1", "container1", compactUID, 0, "") - - // Create mon_group via compact UID. - err := p.PostCreateContainer(context.Background(), pod, ctr) - require.NoError(t, err) - - monDir := p.state.getMonGroupDir(canonicalUID) - require.NotEmpty(t, monDir) - require.NoError(t, os.WriteFile(filepath.Join(monDir, "tasks"), nil, 0644)) - - // StartContainer also using compact UID must find and write to the same mon_group. - ctrWithPid := makeContainer("c1", "container1", compactUID, 77, "") - err = p.StartContainer(context.Background(), pod, ctrWithPid) - require.NoError(t, err) - - data, err := os.ReadFile(filepath.Join(monDir, "tasks")) - require.NoError(t, err) - assert.Equal(t, "77\n", string(data)) -} - -func TestCompactUID_RemovePodSandboxCleansUp(t *testing.T) { - tmpDir := t.TempDir() - p := newTestPlugin(tmpDir) - - compactUID := "a1b2c3d4e5f678901234abcdef567890" - canonicalUID := "a1b2c3d4-e5f6-7890-1234-abcdef567890" - - pod := makePod(compactUID, "default", "test-pod") - ctr := makeContainer("c1", "container1", compactUID, 0, "") - - err := p.PostCreateContainer(context.Background(), pod, ctr) - require.NoError(t, err) - assert.Equal(t, 1, p.state.podCount()) - - // Stopping the last container retains the mon_group. - _, err = p.StopContainer(context.Background(), pod, ctr) - require.NoError(t, err) - assert.Equal(t, 1, p.state.podCount()) - - // Removing the pod sandbox (compact UID) must normalize and clean up. - err = p.RemovePodSandbox(context.Background(), pod) - require.NoError(t, err) - assert.Equal(t, 0, p.state.podCount()) - assert.False(t, p.state.hasPod(canonicalUID)) -} - -func TestRemovePodSandbox_RemovesStateOnRmdirFailure(t *testing.T) { +func TestRemovePodSandbox_RetainsGroupOnRmdirFailure(t *testing.T) { tmpDir := t.TempDir() p := newTestPlugin(tmpDir) podUID := "a1b2c3d4-e5f6-7890-abcd-ef1234567890" @@ -515,24 +439,19 @@ func TestRemovePodSandbox_RemovesStateOnRmdirFailure(t *testing.T) { // Create the mon_group. err := p.PostCreateContainer(context.Background(), pod, ctr) require.NoError(t, err) - assert.Equal(t, 1, p.state.podCount()) + assert.Equal(t, 1, len(p.mgr.List())) - monDir := p.state.getMonGroupDir(podUID) - require.NotEmpty(t, monDir) + monDir := filepath.Join(tmpDir, "mon_groups", podUID) + require.DirExists(t, monDir) - // Put a file inside the mon_group dir so os.Remove fails (dir not empty). + // Put a file inside the mon_group dir so os.Remove (rmdir) would fail. require.NoError(t, os.WriteFile(filepath.Join(monDir, "tasks"), nil, 0644)) - // Stopping the last container retains the mon_group. - _, err = p.StopContainer(context.Background(), pod, ctr) - require.NoError(t, err) - assert.Equal(t, 1, p.state.podCount()) - - // RemovePodSandbox should still drop the pod from state even if rmdir - // fails (the orphaned directory is reaped later by the reconciler). + // RemovePodSandbox attempts removal; with a non-empty dir rmdir fails, + // so the entry remains in the manager (reconciler will retry later). err = p.RemovePodSandbox(context.Background(), pod) - require.NoError(t, err) - assert.Equal(t, 0, p.state.podCount()) + require.NoError(t, err) // handler does not propagate the rmdir error + assert.Equal(t, 1, len(p.mgr.List()), "entry retained when rmdir fails; reconciler will clean") } func TestCheckRuntimeVersion(t *testing.T) { @@ -571,3 +490,89 @@ func TestCheckRuntimeVersion(t *testing.T) { }) } } + +// TestSetConfig_ReloadTearsDownTelemetry verifies that a dynamic setConfig +// reload, after telemetry has started, unregisters the OTel instruments and +// starts fresh telemetry, rather than leaking the old registration. +func TestSetConfig_ReloadTearsDownTelemetry(t *testing.T) { + groups := map[string]map[string]map[string]string{ + "11111111-1111-1111-1111-111111111111": { + "mon_L3_00": {"llc_occupancy": "4096"}, + }, + } + root1 := setupTestResctrl(t, groups) + + p := newTestPlugin(root1) + // Disable Prometheus so telemetry starts without binding a port. + p.config.Telemetry = defaultTelemetryConfig() + p.config.Telemetry.Prometheus.Enabled = false + + require.NoError(t, p.startTelemetry(context.Background())) + oldReg := p.metrics + oldTelem := p.telemetry + require.NotNil(t, oldReg) + require.NotNil(t, oldTelem) + + // Dynamic reconfiguration (same, immutable root), telemetry still port-less. + data := []byte("resctrlPath: " + root1 + "\ntelemetry:\n prometheus:\n enabled: false\n") + require.NoError(t, p.setConfig(data)) + t.Cleanup(func() { + if p.telemetry != nil { + p.telemetry.shutdown(context.Background()) + } + }) + + // Telemetry and its registration were replaced, not leaked. + require.NotNil(t, p.telemetry) + require.NotNil(t, p.metrics) + assert.NotSame(t, oldTelem, p.telemetry) + assert.NotSame(t, oldReg, p.metrics) + + // The old registration was already unregistered; a second call is a no-op. + assert.NoError(t, oldReg.Unregister()) +} + +// TestSetConfig_RejectsRootChange verifies that changing resctrlPath on a +// running plugin (telemetry started) is rejected and the original root is +// retained. +func TestSetConfig_RejectsRootChange(t *testing.T) { + empty := map[string]map[string]map[string]string{} + root1 := setupTestResctrl(t, empty) + root2 := setupTestResctrl(t, empty) + + p := newTestPlugin(root1) + // Bring the plugin up (port-less telemetry) so the immutability guard is + // in force. + p.config.Telemetry = defaultTelemetryConfig() + p.config.Telemetry.Prometheus.Enabled = false + require.NoError(t, p.startTelemetry(context.Background())) + t.Cleanup(func() { + if p.telemetry != nil { + p.telemetry.shutdown(context.Background()) + } + }) + + err := p.setConfig([]byte("resctrlPath: " + root2 + "\ntelemetry:\n prometheus:\n enabled: false\n")) + require.Error(t, err) + assert.Contains(t, err.Error(), "cannot be changed") + assert.Equal(t, root1, p.config.ResctrlPath) +} + +// TestSetConfig_AllowsInitialRootSelection verifies that a non-default +// resctrlPath supplied before the plugin is running (no telemetry, no +// reconciler) is accepted and rebuilds the manager, rather than being rejected +// as a change to a running plugin. +func TestSetConfig_AllowsInitialRootSelection(t *testing.T) { + empty := map[string]map[string]map[string]string{} + root1 := setupTestResctrl(t, empty) + root2 := setupTestResctrl(t, empty) + + // newPlugin-equivalent initial state: config points at root1, no telemetry + // or reconciler running yet. + p := newTestPlugin(root1) + oldMgr := p.mgr + + require.NoError(t, p.setConfig([]byte("resctrlPath: "+root2+"\n"))) + assert.Equal(t, root2, p.config.ResctrlPath) + assert.NotSame(t, oldMgr, p.mgr, "manager should be rebuilt for the new root") +} diff --git a/cmd/plugins/resctrl-mon/resctrl.go b/cmd/plugins/resctrl-mon/resctrl.go deleted file mode 100644 index b4c9f9ae6..000000000 --- a/cmd/plugins/resctrl-mon/resctrl.go +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright The NRI Plugins Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "errors" - "fmt" - "os" - "path/filepath" - "strconv" - "strings" - "syscall" - - "github.com/google/uuid" -) - -const ( - defaultResctrlPath = "/sys/fs/resctrl" - monGroupsDir = "mon_groups" -) - -// resctrlOps handles filesystem operations on the resctrl mount. -type resctrlOps struct { - resctrlPath string -} - -func newResctrlOps(resctrlPath string) *resctrlOps { - return &resctrlOps{ - resctrlPath: resctrlPath, - } -} - -// createMonGroup creates a mon_group directory under the appropriate ctrl_group -// and returns the full path. If rdtClass is empty, the mon_group is created -// under the root resctrl directory. -// -// The kernel assigns an RMID to the new mon_group on mkdir. If no RMIDs are -// available, mkdir returns ENOSPC. -func (r *resctrlOps) createMonGroup(rdtClass, podUID string) (string, error) { - parentDir := r.resctrlPath - if rdtClass != "" { - if !isValidRDTClass(rdtClass) { - return "", fmt.Errorf("invalid RDT class name %q", rdtClass) - } - parentDir = filepath.Join(r.resctrlPath, rdtClass) - } - - // When an RDT class is specified, the ctrl_group must already exist - // (created by an allocation plugin). Do not create it implicitly — - // that would make an unintended ctrl_group in the resctrl filesystem. - if rdtClass != "" { - info, err := os.Stat(parentDir) - if err != nil { - return "", fmt.Errorf("ctrl_group %s does not exist: %w", parentDir, err) - } - if !info.IsDir() { - return "", fmt.Errorf("ctrl_group %s is not a directory", parentDir) - } - } - - monGroupsPath := filepath.Join(parentDir, monGroupsDir) - monGroupDir := filepath.Join(monGroupsPath, podUID) - - // Ensure the mon_groups/ directory exists. On a real resctrl mount - // this is always present. For testing, create it if needed. - if err := os.MkdirAll(monGroupsPath, 0755); err != nil { - return "", fmt.Errorf("mon_groups dir not available at %s: %w", monGroupsPath, err) - } - - // Use Mkdir (not MkdirAll) for the final mon_group directory to - // avoid accidentally creating a ctrl_group if rdtClass is wrong. - if err := os.Mkdir(monGroupDir, 0755); err != nil { - if errors.Is(err, os.ErrExist) { - return monGroupDir, nil - } - if errors.Is(err, syscall.ENOSPC) { - return "", fmt.Errorf("no RMIDs available for pod %s: %w", podUID, err) - } - return "", fmt.Errorf("failed to create mon_group %s: %w", monGroupDir, err) - } - - return monGroupDir, nil -} - -// removeMonGroup removes a mon_group directory. The kernel releases the RMID. -func (r *resctrlOps) removeMonGroup(monGroupDir string) error { - err := os.Remove(monGroupDir) - if err != nil && !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("failed to remove mon_group %s: %w", monGroupDir, err) - } - return nil -} - -// writeTaskPID writes a PID to the mon_group's tasks file. The kernel assigns -// this PID (and all future child processes) to the mon_group's RMID. -func (r *resctrlOps) writeTaskPID(monGroupDir string, pid int) error { - tasksFile := filepath.Join(monGroupDir, "tasks") - f, err := os.OpenFile(tasksFile, os.O_WRONLY, 0) - if err != nil { - return fmt.Errorf("failed to open %s for pid %d: %w", tasksFile, pid, err) - } - defer func() { _ = f.Close() }() - data := []byte(strconv.Itoa(pid) + "\n") - if _, err := f.Write(data); err != nil { - return fmt.Errorf("failed to write pid %d to %s: %w", pid, tasksFile, err) - } - return nil -} - -// cleanOrphanedMonGroups removes mon_group directories that are not tracked -// in the given state. This handles cleanup after a plugin crash/restart. -func (r *resctrlOps) cleanOrphanedMonGroups(state *podState) { - // Scan root-level mon_groups. - r.cleanOrphanedInDir(filepath.Join(r.resctrlPath, monGroupsDir), state) - - // Scan ctrl_group-level mon_groups. - entries, err := os.ReadDir(r.resctrlPath) - if err != nil { - log.Warnf("cleanOrphanedMonGroups: failed to read %s: %v", r.resctrlPath, err) - return - } - for _, entry := range entries { - if !entry.IsDir() { - continue - } - name := entry.Name() - // Skip non-ctrl_group entries. - if name == monGroupsDir || name == "info" || strings.HasPrefix(name, "mon_") { - continue - } - ctrlGroupMonDir := filepath.Join(r.resctrlPath, name, monGroupsDir) - r.cleanOrphanedInDir(ctrlGroupMonDir, state) - } -} - -// cleanOrphanedInDir removes mon_group directories in a specific mon_groups/ -// directory that look like pod UIDs but are not tracked in state. -func (r *resctrlOps) cleanOrphanedInDir(monGroupsPath string, state *podState) { - entries, err := os.ReadDir(monGroupsPath) - if err != nil { - if !errors.Is(err, os.ErrNotExist) { - log.Warnf("failed to read mon_groups directory %s: %v", monGroupsPath, err) - } - return - } - for _, entry := range entries { - if !entry.IsDir() { - continue - } - name := entry.Name() - // Only clean directories that look like pod UIDs. - u, err := uuid.Parse(name) - if err != nil { - continue - } - orphanDir := filepath.Join(monGroupsPath, name) - trackedDir := state.getMonGroupDir(u.String()) - if trackedDir == orphanDir { - // This is the active mon_group for this pod. - continue - } - log.Infof("removing orphaned mon_group %s", orphanDir) - if err := os.Remove(orphanDir); err != nil && !errors.Is(err, os.ErrNotExist) { - log.Warnf("failed to remove orphaned mon_group %s: %v", orphanDir, err) - } - } -} - -// isValidRDTClass returns true if the name is a safe resctrl ctrl_group name. -// It rejects path separators, dot-segments, and empty strings to prevent -// path traversal outside the resctrl mount. -func isValidRDTClass(name string) bool { - if name == "" || name == "." || name == ".." { - return false - } - for _, c := range name { - if c == '/' || c == 0 { - return false - } - } - return true -} diff --git a/cmd/plugins/resctrl-mon/resctrl_test.go b/cmd/plugins/resctrl-mon/resctrl_test.go deleted file mode 100644 index b74aa1c20..000000000 --- a/cmd/plugins/resctrl-mon/resctrl_test.go +++ /dev/null @@ -1,251 +0,0 @@ -// Copyright The NRI Plugins Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestCreateMonGroup_RootClass(t *testing.T) { - tmpDir := t.TempDir() - r := newResctrlOps(tmpDir) - - dir, err := r.createMonGroup("", "pod-uid-1") - require.NoError(t, err) - assert.Equal(t, filepath.Join(tmpDir, "mon_groups", "pod-uid-1"), dir) - - // Directory should exist. - info, err := os.Stat(dir) - require.NoError(t, err) - assert.True(t, info.IsDir()) -} - -func TestCreateMonGroup_WithRDTClass(t *testing.T) { - tmpDir := t.TempDir() - r := newResctrlOps(tmpDir) - require.NoError(t, os.Mkdir(filepath.Join(tmpDir, "BestEffort"), 0755)) - - dir, err := r.createMonGroup("BestEffort", "pod-uid-2") - require.NoError(t, err) - assert.Equal(t, filepath.Join(tmpDir, "BestEffort", "mon_groups", "pod-uid-2"), dir) - - info, err := os.Stat(dir) - require.NoError(t, err) - assert.True(t, info.IsDir()) -} - -func TestCreateMonGroup_MissingCtrlGroup(t *testing.T) { - tmpDir := t.TempDir() - r := newResctrlOps(tmpDir) - - // Attempt to create a mon_group under a non-existent ctrl_group. - _, err := r.createMonGroup("NoSuchClass", "pod-uid-3") - assert.Error(t, err) - assert.Contains(t, err.Error(), "ctrl_group") - - // Verify the ctrl_group was NOT created. - _, err = os.Stat(filepath.Join(tmpDir, "NoSuchClass")) - assert.True(t, os.IsNotExist(err)) -} - -func TestCreateMonGroup_Idempotent(t *testing.T) { - tmpDir := t.TempDir() - r := newResctrlOps(tmpDir) - - dir1, err := r.createMonGroup("", "pod-uid-1") - require.NoError(t, err) - - dir2, err := r.createMonGroup("", "pod-uid-1") - require.NoError(t, err) - - assert.Equal(t, dir1, dir2) -} - -func TestRemoveMonGroup(t *testing.T) { - tmpDir := t.TempDir() - r := newResctrlOps(tmpDir) - - dir, err := r.createMonGroup("", "pod-uid-1") - require.NoError(t, err) - - err = r.removeMonGroup(dir) - require.NoError(t, err) - - _, err = os.Stat(dir) - assert.True(t, os.IsNotExist(err)) -} - -func TestRemoveMonGroup_NotExist(t *testing.T) { - tmpDir := t.TempDir() - r := newResctrlOps(tmpDir) - - err := r.removeMonGroup(filepath.Join(tmpDir, "mon_groups", "nonexistent")) - assert.NoError(t, err) -} - -func TestWriteTaskPID(t *testing.T) { - tmpDir := t.TempDir() - r := newResctrlOps(tmpDir) - - dir, err := r.createMonGroup("", "pod-uid-1") - require.NoError(t, err) - - // In real resctrl, the kernel creates the tasks file when the - // mon_group directory is created. Simulate that here. - tasksFile := filepath.Join(dir, "tasks") - require.NoError(t, os.WriteFile(tasksFile, nil, 0644)) - - err = r.writeTaskPID(dir, 12345) - require.NoError(t, err) - - data, err := os.ReadFile(tasksFile) - require.NoError(t, err) - assert.Equal(t, "12345\n", string(data)) -} - -func TestCleanOrphanedMonGroups(t *testing.T) { - tmpDir := t.TempDir() - r := newResctrlOps(tmpDir) - state := newPodState() - - // Create a mon_group that IS tracked. - trackedUID := "a1b2c3d4-e5f6-7890-abcd-ef1234567890" - dir, err := r.createMonGroup("", trackedUID) - require.NoError(t, err) - state.addPod(trackedUID, dir) - - // Create a mon_group that is NOT tracked (orphan). - orphanUID := "deadbeef-dead-beef-dead-beefdeadbeef" - _, err = r.createMonGroup("", orphanUID) - require.NoError(t, err) - - r.cleanOrphanedMonGroups(state) - - // Tracked should still exist. - _, err = os.Stat(filepath.Join(tmpDir, "mon_groups", trackedUID)) - assert.NoError(t, err) - - // Orphan should be removed. - _, err = os.Stat(filepath.Join(tmpDir, "mon_groups", orphanUID)) - assert.True(t, os.IsNotExist(err)) -} - -func TestCleanOrphanedMonGroups_CtrlGroup(t *testing.T) { - tmpDir := t.TempDir() - r := newResctrlOps(tmpDir) - state := newPodState() - - // Create orphan under a ctrl_group. - orphanUID := "deadbeef-dead-beef-dead-beefdeadbeef" - require.NoError(t, os.Mkdir(filepath.Join(tmpDir, "BestEffort"), 0755)) - _, err := r.createMonGroup("BestEffort", orphanUID) - require.NoError(t, err) - - r.cleanOrphanedMonGroups(state) - - _, err = os.Stat(filepath.Join(tmpDir, "BestEffort", "mon_groups", orphanUID)) - assert.True(t, os.IsNotExist(err)) -} - -func TestCleanOrphanedMonGroups_StaleLocation(t *testing.T) { - tmpDir := t.TempDir() - r := newResctrlOps(tmpDir) - state := newPodState() - - podUID := "a1b2c3d4-e5f6-7890-abcd-ef1234567890" - - // Create a mon_group under BestEffort (simulates previous run). - require.NoError(t, os.Mkdir(filepath.Join(tmpDir, "BestEffort"), 0755)) - _, err := r.createMonGroup("BestEffort", podUID) - require.NoError(t, err) - - // Track the pod at the root class (simulates current run with different RDT class). - rootDir, err := r.createMonGroup("", podUID) - require.NoError(t, err) - state.addPod(podUID, rootDir) - - r.cleanOrphanedMonGroups(state) - - // Root mon_group (tracked) should still exist. - _, err = os.Stat(rootDir) - assert.NoError(t, err) - - // BestEffort mon_group (stale) should be removed. - _, err = os.Stat(filepath.Join(tmpDir, "BestEffort", "mon_groups", podUID)) - assert.True(t, os.IsNotExist(err)) -} - -func TestCleanOrphanedMonGroups_CompactUIDIsOrphaned(t *testing.T) { - tmpDir := t.TempDir() - r := newResctrlOps(tmpDir) - state := newPodState() - - // Simulate: pod is tracked under the canonical dashed UID (as ensureMonGroup stores it). - canonicalUID := "a1b2c3d4-e5f6-7890-abcd-ef1234567890" - canonicalDir, err := r.createMonGroup("", canonicalUID) - require.NoError(t, err) - state.addPod(canonicalUID, canonicalDir) - - // Simulate: a stale compact-form directory left by a previous run (e.g. CRI-O). - compactUID := "a1b2c3d4e5f678901234abcdef567890" - compactDir := filepath.Join(tmpDir, "mon_groups", compactUID) - require.NoError(t, os.Mkdir(compactDir, 0755)) - - r.cleanOrphanedMonGroups(state) - - // Canonical mon_group (tracked) must survive. - _, err = os.Stat(canonicalDir) - assert.NoError(t, err) - - // Compact-named directory is not tracked — must be removed as orphan. - _, err = os.Stat(compactDir) - assert.True(t, os.IsNotExist(err)) -} - -func TestIsValidRDTClass(t *testing.T) { - assert.True(t, isValidRDTClass("BestEffort")) - assert.True(t, isValidRDTClass("Guaranteed")) - assert.True(t, isValidRDTClass("COS1")) - assert.True(t, isValidRDTClass("my-class_v2")) - - assert.False(t, isValidRDTClass("")) - assert.False(t, isValidRDTClass(".")) - assert.False(t, isValidRDTClass("..")) - assert.False(t, isValidRDTClass("../../etc")) - assert.False(t, isValidRDTClass("foo/bar")) - assert.False(t, isValidRDTClass("class\x00name")) -} - -func TestCreateMonGroup_PathTraversal(t *testing.T) { - tmpDir := t.TempDir() - r := newResctrlOps(tmpDir) - - _, err := r.createMonGroup("../../etc", "a1b2c3d4-e5f6-7890-abcd-ef1234567890") - assert.Error(t, err) - assert.Contains(t, err.Error(), "invalid RDT class") - - _, err = r.createMonGroup("foo/bar", "a1b2c3d4-e5f6-7890-abcd-ef1234567890") - assert.Error(t, err) - assert.Contains(t, err.Error(), "invalid RDT class") - - _, err = r.createMonGroup("..", "a1b2c3d4-e5f6-7890-abcd-ef1234567890") - assert.Error(t, err) - assert.Contains(t, err.Error(), "invalid RDT class") -} diff --git a/cmd/plugins/resctrl-mon/state.go b/cmd/plugins/resctrl-mon/state.go deleted file mode 100644 index b3c58d8ab..000000000 --- a/cmd/plugins/resctrl-mon/state.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright The NRI Plugins Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import "sync" - -// podInfo tracks the mon_group directory and container set for a single pod. -type podInfo struct { - monGroupDir string - containers map[string]struct{} // container IDs -} - -// podState tracks all pods with active mon_groups. -type podState struct { - mu sync.Mutex - pods map[string]*podInfo // keyed by pod UID -} - -func newPodState() *podState { - return &podState{ - pods: make(map[string]*podInfo), - } -} - -// addPod registers a new pod with its mon_group directory. -// If the pod already exists, the existing entry is preserved. -func (s *podState) addPod(podUID, monGroupDir string) { - s.mu.Lock() - defer s.mu.Unlock() - if _, ok := s.pods[podUID]; ok { - return - } - s.pods[podUID] = &podInfo{ - monGroupDir: monGroupDir, - containers: make(map[string]struct{}), - } -} - -// addContainer adds a container ID to an existing pod's tracking. -func (s *podState) addContainer(podUID, containerID string) { - s.mu.Lock() - defer s.mu.Unlock() - if info, ok := s.pods[podUID]; ok { - info.containers[containerID] = struct{}{} - } -} - -// removeContainer removes a container ID from a pod's tracking. -func (s *podState) removeContainer(podUID, containerID string) { - s.mu.Lock() - defer s.mu.Unlock() - if info, ok := s.pods[podUID]; ok { - delete(info.containers, containerID) - } else { - log.Warnf("removeContainer: pod %s not tracked (container %s)", podUID, containerID) - } -} - -// removePod removes all tracking for a pod. -func (s *podState) removePod(podUID string) { - s.mu.Lock() - defer s.mu.Unlock() - delete(s.pods, podUID) -} - -// getMonGroupDir returns the mon_group directory for a pod, or empty string. -func (s *podState) getMonGroupDir(podUID string) string { - s.mu.Lock() - defer s.mu.Unlock() - if info, ok := s.pods[podUID]; ok { - return info.monGroupDir - } - return "" -} - -// podHasNoContainers returns true if the pod has no remaining containers. -// Returns true for untracked pods since there is nothing to protect. -func (s *podState) podHasNoContainers(podUID string) bool { - s.mu.Lock() - defer s.mu.Unlock() - if info, ok := s.pods[podUID]; ok { - return len(info.containers) == 0 - } - log.Warnf("podHasNoContainers: pod %s not tracked, treating as empty", podUID) - return true -} - -// hasPod returns true if the pod UID is being tracked. -func (s *podState) hasPod(podUID string) bool { - s.mu.Lock() - defer s.mu.Unlock() - _, ok := s.pods[podUID] - return ok -} - -// podCount returns the number of tracked pods. -func (s *podState) podCount() int { - s.mu.Lock() - defer s.mu.Unlock() - return len(s.pods) -} diff --git a/cmd/plugins/resctrl-mon/telemetry.go b/cmd/plugins/resctrl-mon/telemetry.go new file mode 100644 index 000000000..1ed80371d --- /dev/null +++ b/cmd/plugins/resctrl-mon/telemetry.go @@ -0,0 +1,248 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "net" + "net/http" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" + promexp "go.opentelemetry.io/otel/exporters/prometheus" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" +) + +// telemetryConfig holds the OTel exporter configuration for the plugin. +type telemetryConfig struct { + Prometheus struct { + Enabled bool `json:"enabled"` + ListenAddress string `json:"listenAddress"` + Namespace string `json:"namespace"` + } `json:"prometheus"` + OTLP struct { + Enabled bool `json:"enabled"` + Endpoint string `json:"endpoint"` + Protocol string `json:"protocol"` + Interval string `json:"interval"` + Insecure bool `json:"insecure"` + } `json:"otlp"` + PerfCounters struct { + Enabled bool `json:"enabled"` + Include []string `json:"include"` + Exclude []string `json:"exclude"` + } `json:"perfCounters"` + ResourceAttributes map[string]string `json:"resourceAttributes"` +} + +// telemetryState holds the running telemetry components for graceful shutdown. +type telemetryState struct { + provider *metric.MeterProvider + server *http.Server + promListener net.Listener +} + +// defaultTelemetryConfig returns the config defaults (Prometheus on :9100). +func defaultTelemetryConfig() telemetryConfig { + var cfg telemetryConfig + cfg.Prometheus.Enabled = true + cfg.Prometheus.ListenAddress = ":9100" + // Match the chart/sample/README default: OTLP uses a plaintext connection + // unless the operator explicitly opts into TLS. + cfg.OTLP.Insecure = true + return cfg +} + +// validateTelemetryConfig checks the telemetry config for invalid combinations. +func validateTelemetryConfig(cfg *telemetryConfig) error { + if cfg.OTLP.Enabled && cfg.OTLP.Endpoint == "" { + return fmt.Errorf("telemetry: otlp.enabled requires a non-empty endpoint") + } + if cfg.OTLP.Protocol == "" { + cfg.OTLP.Protocol = "grpc" + } + if cfg.OTLP.Protocol != "grpc" && cfg.OTLP.Protocol != "http" { + return fmt.Errorf("telemetry: otlp.protocol must be \"grpc\" or \"http\", got %q", cfg.OTLP.Protocol) + } + if cfg.OTLP.Interval == "" { + cfg.OTLP.Interval = "15s" + } + if _, err := time.ParseDuration(cfg.OTLP.Interval); err != nil { + return fmt.Errorf("telemetry: otlp.interval %q: %w", cfg.OTLP.Interval, err) + } + if len(cfg.PerfCounters.Include) > 0 && len(cfg.PerfCounters.Exclude) > 0 { + return fmt.Errorf("telemetry: perfCounters.include and perfCounters.exclude are mutually exclusive") + } + if cfg.Prometheus.ListenAddress == "" { + cfg.Prometheus.ListenAddress = ":9100" + } + return nil +} + +// newTelemetry creates the MeterProvider with configured exporters. +func newTelemetry(ctx context.Context, cfg telemetryConfig) (*telemetryState, error) { + var opts []metric.Option + + res, err := resource.New(ctx, + resource.WithAttributes(resourceAttrs(cfg.ResourceAttributes)...), + ) + if err != nil { + return nil, fmt.Errorf("telemetry: failed to create resource: %w", err) + } + opts = append(opts, metric.WithResource(res)) + + state := &telemetryState{} + + if cfg.Prometheus.Enabled { + reg := prometheus.NewRegistry() + peOpts := []promexp.Option{promexp.WithRegisterer(reg)} + if cfg.Prometheus.Namespace != "" { + peOpts = append(peOpts, promexp.WithNamespace(cfg.Prometheus.Namespace)) + } + pe, err := promexp.New(peOpts...) + if err != nil { + return nil, fmt.Errorf("telemetry: prometheus exporter: %w", err) + } + opts = append(opts, metric.WithReader(pe)) + + // Bind synchronously so a failure (e.g. address already in use) is + // returned to the caller instead of only surfacing asynchronously from + // the serving goroutine, which would leave the endpoint advertised for + // scraping with nothing actually listening. + ln, err := net.Listen("tcp", cfg.Prometheus.ListenAddress) + if err != nil { + return nil, fmt.Errorf("telemetry: prometheus listen on %s: %w", cfg.Prometheus.ListenAddress, err) + } + state.promListener = ln + + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{})) + state.server = &http.Server{Addr: cfg.Prometheus.ListenAddress, Handler: mux} + } + + if cfg.OTLP.Enabled { + interval, _ := time.ParseDuration(cfg.OTLP.Interval) + exp, err := newOTLPExporter(ctx, cfg) + if err != nil { + if state.promListener != nil { + _ = state.promListener.Close() + } + return nil, fmt.Errorf("telemetry: OTLP exporter: %w", err) + } + opts = append(opts, metric.WithReader( + metric.NewPeriodicReader(exp, metric.WithInterval(interval)), + )) + log.Infof("telemetry: OTLP push enabled → %s (%s, interval=%s)", + cfg.OTLP.Endpoint, cfg.OTLP.Protocol, cfg.OTLP.Interval) + } + + state.provider = metric.NewMeterProvider(opts...) + + // All fallible setup has succeeded; only now start serving the (already + // bound) Prometheus listener. + if state.promListener != nil { + go func() { + if err := state.server.Serve(state.promListener); err != nil && err != http.ErrServerClosed { + log.Warnf("telemetry: prometheus server error: %v", err) + } + }() + log.Infof("telemetry: Prometheus endpoint listening on %s/metrics", state.promListener.Addr()) + } + + return state, nil +} + +// shutdown gracefully shuts down the telemetry stack. +func (t *telemetryState) shutdown(ctx context.Context) { + if t.provider != nil { + if err := t.provider.Shutdown(ctx); err != nil { + log.Warnf("telemetry: provider shutdown: %v", err) + } + } + if t.server != nil { + if err := t.server.Shutdown(ctx); err != nil { + log.Warnf("telemetry: http server shutdown: %v", err) + } + } +} + +// newOTLPExporter creates a gRPC or HTTP OTLP metric exporter. +func newOTLPExporter(ctx context.Context, cfg telemetryConfig) (metric.Exporter, error) { + switch cfg.OTLP.Protocol { + case "http": + opts := []otlpmetrichttp.Option{ + otlpmetrichttp.WithEndpoint(cfg.OTLP.Endpoint), + } + if cfg.OTLP.Insecure { + opts = append(opts, otlpmetrichttp.WithInsecure()) + } + return otlpmetrichttp.New(ctx, opts...) + default: // grpc + opts := []otlpmetricgrpc.Option{ + otlpmetricgrpc.WithEndpoint(cfg.OTLP.Endpoint), + } + if cfg.OTLP.Insecure { + opts = append(opts, otlpmetricgrpc.WithInsecure()) + } + return otlpmetricgrpc.New(ctx, opts...) + } +} + +// resourceAttrs builds OTel resource attributes from the config map. +func resourceAttrs(m map[string]string) []attribute.KeyValue { + attrs := []attribute.KeyValue{ + semconv.ServiceName("nri-resctrl-mon"), + } + for k, v := range m { + if k == "service.name" { + // Override the default service name. + attrs[0] = semconv.ServiceName(v) + continue + } + attrs = append(attrs, attribute.String(k, v)) + } + return attrs +} + +// startTelemetry initializes the MeterProvider and registers metrics instruments. +func (p *plugin) startTelemetry(ctx context.Context) error { + cfg := p.config.Telemetry + if err := validateTelemetryConfig(&cfg); err != nil { + return err + } + state, err := newTelemetry(ctx, cfg) + if err != nil { + return err + } + + meter := state.provider.Meter("nri-resctrl-mon") + reg, err := setupMetrics(p.mgr, cfg, p.config.ResctrlPath, meter) + if err != nil { + state.shutdown(ctx) + return fmt.Errorf("metrics registration: %w", err) + } + // Publish only after registration succeeds; otherwise a failed setupMetrics + // would leave p.telemetry non-nil and a later Configure would skip startup. + p.telemetry = state + p.metrics = reg + return nil +} diff --git a/cmd/plugins/resctrl-mon/telemetry_test.go b/cmd/plugins/resctrl-mon/telemetry_test.go new file mode 100644 index 000000000..c856d701a --- /dev/null +++ b/cmd/plugins/resctrl-mon/telemetry_test.go @@ -0,0 +1,326 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "io" + "net/http" + "os" + "path/filepath" + "testing" + "time" + + "github.com/intel/goresctrl/pkg/monitor" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// setupTestResctrl creates a minimal resctrl-like filesystem for testing. +// Returns the root path and a cleanup function. +func setupTestResctrl(t *testing.T, groups map[string]map[string]map[string]string) string { + t.Helper() + root := t.TempDir() + monGroupsDir := filepath.Join(root, "mon_groups") + require.NoError(t, os.MkdirAll(monGroupsDir, 0o755)) + + // Collect all domains/counters to also create root-level mon_data + // (used by RegisterOTelInstruments to discover available instruments). + allDomains := make(map[string]map[string]string) + + for groupName, domains := range groups { + groupDir := filepath.Join(monGroupsDir, groupName) + require.NoError(t, os.MkdirAll(groupDir, 0o755)) + // Create tasks file. + require.NoError(t, os.WriteFile(filepath.Join(groupDir, "tasks"), []byte(""), 0o644)) + monDataDir := filepath.Join(groupDir, "mon_data") + for domain, counters := range domains { + domDir := filepath.Join(monDataDir, domain) + require.NoError(t, os.MkdirAll(domDir, 0o755)) + for file, value := range counters { + require.NoError(t, os.WriteFile(filepath.Join(domDir, file), []byte(value+"\n"), 0o644)) + } + // Merge into allDomains. + if allDomains[domain] == nil { + allDomains[domain] = make(map[string]string) + } + for file, value := range counters { + allDomains[domain][file] = value + } + } + } + + // Create root-level mon_data for instrument discovery. + rootMonData := filepath.Join(root, "mon_data") + for domain, counters := range allDomains { + domDir := filepath.Join(rootMonData, domain) + require.NoError(t, os.MkdirAll(domDir, 0o755)) + for file, value := range counters { + require.NoError(t, os.WriteFile(filepath.Join(domDir, file), []byte(value+"\n"), 0o644)) + } + } + + return root +} + +func TestTelemetryPrometheusEndpoint(t *testing.T) { + podUID := "12345678-1234-1234-1234-123456789abc" + root := setupTestResctrl(t, map[string]map[string]map[string]string{ + podUID: { + "mon_L3_00": { + "llc_occupancy": "4096", + "mbm_local_bytes": "1000000", + "mbm_total_bytes": "2000000", + }, + "mon_PERF_PKG_00": { + "core_energy": "54446119.644974", + "activity": "12345.6789", + "c6_res": "9999", + }, + }, + }) + + mgr, err := monitor.New(monitor.Options{ + ResctrlRoot: root, + KeyValidator: monitor.PodUIDValidator, + KeyCanonicalizer: monitor.CanonicalizePodUID, + }) + require.NoError(t, err) + _, err = mgr.EnsureGroup(podUID, "") + require.NoError(t, err) + + // Bind an ephemeral port to avoid conflicts on shared CI runners. + cfg := defaultTelemetryConfig() + cfg.Prometheus.ListenAddress = "127.0.0.1:0" + cfg.PerfCounters.Enabled = false // default: suppress perf counters + + state, err := newTelemetry(context.Background(), cfg) + require.NoError(t, err) + defer state.shutdown(context.Background()) + + meter := state.provider.Meter("nri-resctrl-mon-test") + _, err = setupMetrics(mgr, cfg, root, meter) + require.NoError(t, err) + + // Scrape the actual address the listener bound to. + addr := state.promListener.Addr().String() + + // Wait for server to be ready. + time.Sleep(50 * time.Millisecond) + + resp, err := http.Get("http://" + addr + "/metrics") + require.NoError(t, err) + defer func() { _ = resp.Body.Close() }() + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + metrics := string(body) + + // Verify core instruments are registered and produce correct Prometheus names. + // Instrument names are derived from domain + counter: + // mon_L3_00/llc_occupancy → l3.llc.occupancy (unit By) → l3_llc_occupancy_bytes + // mon_L3_00/mbm_local_bytes → l3.mbm.local.bytes (unit By) → l3_mbm_local_bytes_total + // mon_L3_00/mbm_total_bytes → l3.mbm.total.bytes (unit By) → l3_mbm_bytes_total + // (the otlptranslator collapses the semantic "total" into the counter + // "_total" suffix; the local counter above disambiguates it) + // mon_PERF_PKG_00/core_energy → perf.core.energy (unit J) → perf_core_energy_joules_total + // mon_PERF_PKG_00/activity → perf.activity (unit farads) → perf_activity_farads_total + assert.Contains(t, metrics, "l3_llc_occupancy_bytes") + assert.Contains(t, metrics, "l3_mbm_local_bytes_total") + assert.Contains(t, metrics, "l3_mbm_bytes_total") + assert.Contains(t, metrics, "perf_core_energy_joules_total") + assert.Contains(t, metrics, "perf_activity_farads_total") + + // Verify float64 fidelity through the full pipeline. + assert.Contains(t, metrics, "5.4446119644974e+07", + "core_energy float64 value must survive OTel→Prometheus rendering") +} + +func TestFloat64Fidelity(t *testing.T) { + podUID := "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + root := setupTestResctrl(t, map[string]map[string]map[string]string{ + podUID: { + "mon_PERF_PKG_00": { + "core_energy": "54446119.644974", + }, + }, + }) + + mgr, err := monitor.New(monitor.Options{ + ResctrlRoot: root, + KeyValidator: monitor.PodUIDValidator, + KeyCanonicalizer: monitor.CanonicalizePodUID, + }) + require.NoError(t, err) + _, err = mgr.EnsureGroup(podUID, "") + require.NoError(t, err) + + readings, err := mgr.ReadCounters(podUID) + require.NoError(t, err) + require.Len(t, readings, 1) + assert.Equal(t, 54446119.644974, readings[0].Value, + "float64 value must preserve kernel precision without integer truncation") +} + +func TestPerfCountersGate(t *testing.T) { + podUID := "11111111-2222-3333-4444-555555555555" + root := setupTestResctrl(t, map[string]map[string]map[string]string{ + podUID: { + "mon_PERF_PKG_00": { + "core_energy": "100.5", + "activity": "50.0", + "c6_res": "9999", + "unhalted_core_cycles": "123456", + }, + "mon_L3_00": { + "llc_occupancy": "8192", + }, + }, + }) + + mgr, err := monitor.New(monitor.Options{ + ResctrlRoot: root, + KeyValidator: monitor.PodUIDValidator, + KeyCanonicalizer: monitor.CanonicalizePodUID, + }) + require.NoError(t, err) + _, err = mgr.EnsureGroup(podUID, "") + require.NoError(t, err) + + readings, err := mgr.ReadCounters(podUID) + require.NoError(t, err) + + t.Run("disabled suppresses perf-only counters", func(t *testing.T) { + cfg := defaultTelemetryConfig() + cfg.PerfCounters.Enabled = false + filter := perfCounterFilter(cfg) + + var allowed []string + for _, r := range readings { + if filter(r) { + allowed = append(allowed, r.Name) + } + } + // core_energy, activity (from PERF_PKG) + llc_occupancy (from L3) should pass. + assert.Contains(t, allowed, "core_energy") + assert.Contains(t, allowed, "activity") + assert.Contains(t, allowed, "llc_occupancy") + // c6_res and unhalted_core_cycles should be blocked. + assert.NotContains(t, allowed, "c6_res") + assert.NotContains(t, allowed, "unhalted_core_cycles") + }) + + t.Run("enabled allows all perf counters", func(t *testing.T) { + cfg := defaultTelemetryConfig() + cfg.PerfCounters.Enabled = true + filter := perfCounterFilter(cfg) + + var allowed []string + for _, r := range readings { + if filter(r) { + allowed = append(allowed, r.Name) + } + } + assert.Contains(t, allowed, "c6_res") + assert.Contains(t, allowed, "unhalted_core_cycles") + }) + + t.Run("include list filters", func(t *testing.T) { + cfg := defaultTelemetryConfig() + cfg.PerfCounters.Enabled = true + cfg.PerfCounters.Include = []string{"unhalted_*"} + filter := perfCounterFilter(cfg) + + var allowed []string + for _, r := range readings { + if filter(r) { + allowed = append(allowed, r.Name) + } + } + assert.Contains(t, allowed, "unhalted_core_cycles") + assert.Contains(t, allowed, "core_energy") // always allowed (core AET) + assert.NotContains(t, allowed, "c6_res") // not in include list + }) +} + +func TestControlGroupOf(t *testing.T) { + tests := []struct { + root string + path string + want string + }{ + {"/sys/fs/resctrl", "/sys/fs/resctrl/mon_groups/abc-123", ""}, + {"/sys/fs/resctrl", "/sys/fs/resctrl/COS1/mon_groups/abc-123", "COS1"}, + {"/sys/fs/resctrl", "/sys/fs/resctrl/my-class/mon_groups/abc-123", "my-class"}, + {"/mnt/rdt", "/mnt/rdt/mon_groups/abc-123", ""}, + {"/mnt/rdt", "/mnt/rdt/COS1/mon_groups/abc-123", "COS1"}, + } + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + assert.Equal(t, tt.want, controlGroupOf(tt.root, tt.path)) + }) + } +} + +func TestValidateTelemetryConfig(t *testing.T) { + t.Run("valid defaults", func(t *testing.T) { + cfg := defaultTelemetryConfig() + assert.NoError(t, validateTelemetryConfig(&cfg)) + }) + + t.Run("otlp enabled requires endpoint", func(t *testing.T) { + cfg := defaultTelemetryConfig() + cfg.OTLP.Enabled = true + assert.Error(t, validateTelemetryConfig(&cfg)) + }) + + t.Run("invalid protocol", func(t *testing.T) { + cfg := defaultTelemetryConfig() + cfg.OTLP.Enabled = true + cfg.OTLP.Endpoint = "localhost:4317" + cfg.OTLP.Protocol = "websocket" + assert.Error(t, validateTelemetryConfig(&cfg)) + }) + + t.Run("include and exclude mutually exclusive", func(t *testing.T) { + cfg := defaultTelemetryConfig() + cfg.PerfCounters.Include = []string{"c6_res"} + cfg.PerfCounters.Exclude = []string{"c1_res"} + assert.Error(t, validateTelemetryConfig(&cfg)) + }) +} + +func TestInstrumentNaming(t *testing.T) { + // Verify the library's derived naming matches expectations. + t.Run("L3 counters match rdt convention", func(t *testing.T) { + assert.Equal(t, "l3.llc.occupancy", monitor.InstrumentName("mon_L3_00", "llc_occupancy")) + assert.Equal(t, "l3.mbm.local.bytes", monitor.InstrumentName("mon_L3_00", "mbm_local_bytes")) + assert.Equal(t, "l3.mbm.total.bytes", monitor.InstrumentName("mon_L3_00", "mbm_total_bytes")) + }) + + t.Run("PERF_PKG counters", func(t *testing.T) { + assert.Equal(t, "perf.core.energy", monitor.InstrumentName("mon_PERF_PKG_00", "core_energy")) + assert.Equal(t, "perf.activity", monitor.InstrumentName("mon_PERF_PKG_00", "activity")) + assert.Equal(t, "perf.c1.res", monitor.InstrumentName("mon_PERF_PKG_00", "c1_res")) + }) +} + +func TestMatchGlob(t *testing.T) { + assert.True(t, matchGlob("unhalted_*", "unhalted_core_cycles")) + assert.True(t, matchGlob("unhalted_*", "unhalted_ref_cycles")) + assert.False(t, matchGlob("unhalted_*", "c6_res")) + assert.True(t, matchGlob("c6_res", "c6_res")) + assert.True(t, matchGlob("*_bytes", "mbm_local_bytes")) +} diff --git a/deployment/helm/resctrl-mon/README.md b/deployment/helm/resctrl-mon/README.md index b77809acf..e223879d5 100644 --- a/deployment/helm/resctrl-mon/README.md +++ b/deployment/helm/resctrl-mon/README.md @@ -121,3 +121,70 @@ customize with their own values, along with the default values. | `affinity` | [] | specify node affinity | | `nodeSelector` | [] | specify node selector labels | | `podPriorityClassNodeCritical` | true | enable [marking Pod as node critical](https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/#marking-pod-as-critical) | + +### Telemetry options + +| Name | Default | Description | +| -------------------------------------- | --------- | ------------------------------------------------------------------ | +| `telemetry.prometheus.enabled` | `true` | expose a `/metrics` Prometheus endpoint | +| `telemetry.prometheus.listenAddress` | `":9100"` | address:port for the Prometheus HTTP listener | +| `telemetry.prometheus.scrapeInterval` | `"15s"` | recommended scrape interval (set via pod annotation hint) | +| `telemetry.prometheus.namespace` | `""` | Prometheus metric prefix (empty = no prefix, e.g. `l3_*`/`perf_*`) | +| `telemetry.otlp.enabled` | `false` | push metrics via OTLP | +| `telemetry.otlp.endpoint` | `""` | OTLP receiver endpoint (e.g. `otel-collector-resctrl:4317`) | +| `telemetry.otlp.protocol` | `grpc` | `grpc` or `http` | +| `telemetry.otlp.interval` | `15s` | OTLP export interval | +| `telemetry.otlp.insecure` | `true` | disable TLS for OTLP connection | +| `telemetry.perfCounters.enabled` | `false` | gate `rdt=perf` counters (c1_res, stalls_*, etc.) | +| `telemetry.perfCounters.include` | `[]` | glob patterns for counters to include | +| `telemetry.perfCounters.exclude` | `[]` | glob patterns for counters to exclude | +| `telemetry.resourceAttributes` | `{}` | static OTel resource attributes added to all metrics | + +## Prometheus Integration + +The DaemonSet pods are annotated with `prometheus.io/scrape: "true"` so that +standard Prometheus service-discovery configurations will pick them up +automatically. The `prometheus.io/interval` annotation is set to the +configured `telemetry.prometheus.scrapeInterval` (default 15s) as a hint, +but note that most Prometheus deployments do not honor this annotation +without additional relabel configuration. + +If you need a specific scrape interval, configure a dedicated scrape job +in your Prometheus configuration with the desired `scrape_interval`. + +## Runtime Requirements + +| Component | Minimum Version | Notes | +| ---------------- | --------------- | --------------------------------------------------------------------- | +| Linux kernel | 5.x+ | CMT/MBM on 5.x+; AET perf/energy counters need `rdt=perf` (pending upstream). | +| containerd | 1.7.0+ | NRI support required. | +| CRI-O | 1.36.0+ | Provides container PIDs via NRI `LinuxContainer.Pid`. | +| Kubernetes | 1.24+ | DaemonSet and NRI socket conventions. | +| CPU | Intel RDT | CMT/MBM for bandwidth/LLC counters; AET for energy/perf counters. | + +### Kernel feature matrix + +| Counter family | Kernel Kconfig | Available since | +| ------------------------------- | ----------------------------- | --------------- | +| `llc_occupancy`, `mbm_*` | `CONFIG_X86_CPU_RESCTRL` | 5.x | +| `c1_res`, `stalls_*`, `energy_*` | `CONFIG_X86_CPU_RESCTRL` + `rdt=perf` boot param | pending (under review) | + +> **Note:** `rdt=perf` kernel support is still under review upstream and is not +> yet part of a released kernel. The "Available since" version for these +> counters is TBD and will be recorded once the change lands. + +## Optional: OTel Collector sidecar + +When using OTLP push mode (`telemetry.otlp.enabled=true`), you may deploy an +OTel Collector agent to receive, enrich, and fan out the metrics. Reference +manifests are provided in `optional/`: + +```sh +kubectl apply -f optional/otel-collector-rbac.yaml +kubectl apply -f optional/otel-collector-agent.yaml +``` + +The reference config uses the `k8sattributes` processor to attach pod/namespace +labels and a Prometheus exporter on port 8889. Customize +`otel-collector-agent.yaml` to add additional exporters (e.g. `otlphttp` to a +remote backend). diff --git a/deployment/helm/resctrl-mon/optional/grafana-resctrl-perf-counters.json b/deployment/helm/resctrl-mon/optional/grafana-resctrl-perf-counters.json new file mode 100644 index 000000000..d3cd05be7 --- /dev/null +++ b/deployment/helm/resctrl-mon/optional/grafana-resctrl-perf-counters.json @@ -0,0 +1,1010 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Prometheus", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(kube_pod_info, pod)", + "hide": 0, + "includeAll": true, + "label": "Pod", + "multi": true, + "name": "pod_name", + "query": { + "qryType": 1, + "query": "label_values(kube_pod_info, pod)" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(kube_pod_info, namespace)", + "hide": 0, + "includeAll": true, + "label": "Namespace", + "multi": true, + "name": "namespace", + "query": { + "qryType": 1, + "query": "label_values(kube_pod_info, namespace)" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + } + ] + }, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total power across all monitored pods", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#1a5cff", + "value": null + }, + { + "color": "#22d66a", + "value": 0 + }, + { + "color": "#f0a030", + "value": 100 + }, + { + "color": "#ff5c5c", + "value": 200 + } + ] + }, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate({__name__=\"perf_core_energy_joules_total\", resctrl_group_source=\"pod\"}[$__rate_interval]))", + "interval": "5s", + "legendFormat": "{{k8s_pod_uid}}", + "refId": "A" + } + ], + "title": "Total Pod Power", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total CPU activity rate across all monitored pods", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#1a5cff", + "value": null + }, + { + "color": "#22d66a", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 5 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate({__name__=\"perf_activity_farads_total\", resctrl_group_source=\"pod\"}[$__rate_interval]))", + "interval": "5s", + "legendFormat": "{{k8s_pod_uid}}", + "refId": "A" + } + ], + "title": "Total Activity Rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total unhalted core cycles/sec across all monitored pods", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#1a5cff", + "value": null + }, + { + "color": "#22d66a", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 9 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate({__name__=\"perf_unhalted_core_cycles_total\", resctrl_group_source=\"pod\"}[$__rate_interval]))", + "interval": "5s", + "legendFormat": "{{k8s_pod_uid}}", + "refId": "A" + } + ], + "title": "Total Core Cycles/s", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total retired micro-ops/sec across all monitored pods", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#1a5cff", + "value": null + }, + { + "color": "#22d66a", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 13 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate({__name__=\"perf_uops_retired_total\", resctrl_group_source=\"pod\"}[$__rate_interval]))", + "interval": "5s", + "legendFormat": "{{k8s_pod_uid}}", + "refId": "A" + } + ], + "title": "Total \u00b5ops Retired/s", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 10, + "title": "Power (Energy Rate)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Per-pod power consumption derived from Intel AET energy counter", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "lineWidth": 1, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (pod) (\n rate({__name__=\"perf_core_energy_joules_total\"}[$__rate_interval])\n * on(k8s_pod_uid) group_left(pod, namespace)\nlabel_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\")\n)", + "interval": "5s", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Pod Power (Watts)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 20, + "title": "Activity Rate", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Per-pod CPU activity rate (capacitance proxy from AET)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "lineWidth": 1, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 21, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (pod) (\n rate({__name__=\"perf_activity_farads_total\"}[$__rate_interval])\n * on(k8s_pod_uid) group_left(pod, namespace)\nlabel_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\")\n)", + "interval": "5s", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Pod Activity Rate (Farads/s)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 30, + "title": "Frequency Scaling Factor", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Ratio of unhalted core cycles to reference cycles per pod. 1.0 = base frequency, >1.0 = turbo, <1.0 = throttled.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "scaling factor (1.0 = base freq)", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "lineWidth": 1, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "green", + "value": 1.0 + } + ] + }, + "unit": "none", + "min": 0, + "decimals": 3 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 31, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (pod) (\n (\n sum by (k8s_pod_uid) (rate({__name__=\"perf_unhalted_core_cycles_total\"}[$__rate_interval]))\n / clamp_min(sum by (k8s_pod_uid) (rate({__name__=\"perf_unhalted_ref_cycles_total\"}[$__rate_interval])), 1)\n )\n * on(k8s_pod_uid) group_left(pod, namespace)\n label_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\")\n)", + "interval": "5s", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Frequency Scaling Factor (core/ref cycles ratio)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 44 + }, + "id": 40, + "title": "\u00b5ops Retired", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Per-pod retired micro-operations per second", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "lineWidth": 1, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 45 + }, + "id": 41, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (pod) (\n rate({__name__=\"perf_uops_retired_total\"}[$__rate_interval])\n * on(k8s_pod_uid) group_left(pod, namespace)\nlabel_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\")\n)", + "interval": "5s", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "\u00b5ops Retired/s", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 53 + }, + "id": 50, + "title": "C-state Residency", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Per-pod C1 residency rate", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "avg cores in C1", + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "lineWidth": 1, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 54 + }, + "id": 51, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (pod) (\n rate({__name__=\"perf_c1_res_total\"}[$__rate_interval])\n * on(k8s_pod_uid) group_left(pod, namespace)\nlabel_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\")\n)", + "interval": "5s", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "C1 Residency Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Per-pod C6 residency rate", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "avg cores in C6", + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "lineWidth": 1, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 62 + }, + "id": 52, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (pod) (\n rate({__name__=\"perf_c6_res_total\"}[$__rate_interval])\n * on(k8s_pod_uid) group_left(pod, namespace)\nlabel_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\")\n)", + "interval": "5s", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "C6 Residency Rate", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 70 + }, + "id": 60, + "title": "LLC Stalls", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Per-pod LLC hit stalls per second", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "lineWidth": 1, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 71 + }, + "id": 61, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (pod) (\n rate({__name__=\"perf_stalls_llc_hit_total\"}[$__rate_interval])\n * on(k8s_pod_uid) group_left(pod, namespace)\nlabel_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\")\n)", + "interval": "5s", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "LLC Hit Stalls/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Per-pod LLC miss stalls per second", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "lineWidth": 1, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 79 + }, + "id": 62, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (pod) (\n rate({__name__=\"perf_stalls_llc_miss_total\"}[$__rate_interval])\n * on(k8s_pod_uid) group_left(pod, namespace)\nlabel_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\")\n)", + "interval": "5s", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "LLC Miss Stalls/s", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": [ + "aet", + "energy", + "kubernetes", + "resctrl", + "perf" + ], + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Kubernetes Pod Perf Counters (Intel AET / resctrl-mon)", + "uid": "resctrl-perf-counters", + "version": 3 +} diff --git a/deployment/helm/resctrl-mon/optional/grafana-resctrl-pod-energy.json b/deployment/helm/resctrl-mon/optional/grafana-resctrl-pod-energy.json new file mode 100644 index 000000000..5af2fdf9d --- /dev/null +++ b/deployment/helm/resctrl-mon/optional/grafana-resctrl-pod-energy.json @@ -0,0 +1,1088 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Sum of power consumed by all monitored pods (from Intel AET via resctrl)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#1a5cff", + "value": null + }, + { + "color": "#22d66a", + "value": 0 + }, + { + "color": "#f0a030", + "value": 100 + }, + { + "color": "#ff5c5c", + "value": 200 + } + ] + }, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate({__name__=\"perf_core_energy_joules_total\", resctrl_group_source=\"pod\"}[$__rate_interval]))", + "legendFormat": "Total Pod Power", + "refId": "A", + "interval": "5s" + } + ], + "title": "Total Pod Power", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Energy consumed by all monitored pods in the displayed time window (Wh)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#22d66a", + "value": null + } + ] + }, + "unit": "watth" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(increase({__name__=\"perf_core_energy_joules_total\", resctrl_group_source=\"pod\"}[$__range])) / 3600", + "legendFormat": "Total Energy", + "refId": "A", + "interval": "5s" + } + ], + "title": "Total Pod Energy", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Percentage of total activity attributable to monitored pods (AET activity counters, measured in Farads)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#22d66a", + "value": null + }, + { + "color": "#f0a030", + "value": 50 + }, + { + "color": "#ff5c5c", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate({__name__=\"perf_activity_farads_total\", resctrl_group_source=\"pod\"}[$__rate_interval])) / sum(rate({__name__=\"perf_activity_farads_total\"}[$__rate_interval])) * 100", + "legendFormat": "Pod-Attributed", + "refId": "A", + "interval": "5s" + } + ], + "title": "Pod-Attributed Activity (share of observed)", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 101, + "title": "Per-Pod Power", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Power consumption per pod from Intel AET core_energy counters", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Power (W)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 16, + "x": 0, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (pod) (rate({__name__=\"perf_core_energy_joules_total\", resctrl_group_source=\"pod\"}[$__rate_interval]) * on(k8s_pod_uid) group_left(pod, namespace) label_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{namespace=~\"$namespace\"}[$__range])), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\"))", + "legendFormat": "{{pod}}", + "refId": "A", + "interval": "5s" + } + ], + "title": "Per-Pod Power Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total energy (Joules) consumed by each workload over the selected range (donut chart). Pods sharing a common name are summed; pods that started and stopped within the range are included.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "joule" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 8 + }, + "id": 5, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "bottom", + "values": [ + "value", + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (workload) (increase({__name__=\"perf_core_energy_joules_total\", resctrl_group_source=\"pod\"}[$__range]) * on(k8s_pod_uid) group_left(workload) label_replace(label_replace(label_replace(label_replace(max by (uid, pod, namespace, created_by_name) (last_over_time(kube_pod_info{namespace=~\"$namespace\"}[$__range])), \"workload\", \"$1\", \"pod\", \"(.+)\"), \"workload\", \"$1\", \"created_by_name\", \"(.+)\"), \"workload\", \"$1\", \"workload\", \"(.+?)-[bcdfghjklmnpqrstvwxz2-9]{6,10}$\"), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\"))", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "title": "Energy Breakdown by Pod", + "type": "piechart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 102, + "title": "Per-Pod Activity", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Activity rate (F/s) per pod from AET activity counters. Activity is measured in Farads and represents frequency-independent work done.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Activity (F/s)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": " F/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 16, + "x": 0, + "y": 19 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (pod) (rate({__name__=\"perf_activity_farads_total\", resctrl_group_source=\"pod\"}[$__rate_interval]) * on(k8s_pod_uid) group_left(pod, namespace) label_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{namespace=~\"$namespace\"}[$__range])), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\"))", + "legendFormat": "{{pod}}", + "refId": "A", + "interval": "5s" + } + ], + "title": "Per-Pod Activity Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total activity (Farads) accumulated by each workload over the selected range (donut chart). Activity is frequency-independent work. Pods sharing a common name are summed; pods that started and stopped within the range are included.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": " F" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 7, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "bottom", + "values": [ + "value", + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (workload) (increase({__name__=\"perf_activity_farads_total\", resctrl_group_source=\"pod\"}[$__range]) * on(k8s_pod_uid) group_left(workload) label_replace(label_replace(label_replace(label_replace(max by (uid, pod, namespace, created_by_name) (last_over_time(kube_pod_info{namespace=~\"$namespace\"}[$__range])), \"workload\", \"$1\", \"pod\", \"(.+)\"), \"workload\", \"$1\", \"created_by_name\", \"(.+)\"), \"workload\", \"$1\", \"workload\", \"(.+?)-[bcdfghjklmnpqrstvwxz2-9]{6,10}$\"), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\"))", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "title": "Activity Breakdown by Pod", + "type": "piechart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 103, + "title": "Pod Details", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "All monitored pods with their current power draw, activity rate, and cumulative energy", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "fieldMinMax": true + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Pod" + }, + "properties": [ + { + "id": "custom.width", + "value": 280 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Namespace" + }, + "properties": [ + { + "id": "custom.width", + "value": 120 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Power (W)" + }, + "properties": [ + { + "id": "unit", + "value": "watt" + }, + { + "id": "decimals", + "value": 3 + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge", + "valueDisplayMode": "text" + } + }, + { + "id": "color", + "value": { + "mode": "continuous-GrYlRd" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Activity (F/s)" + }, + "properties": [ + { + "id": "unit", + "value": " F/s" + }, + { + "id": "decimals", + "value": 3 + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge", + "valueDisplayMode": "text" + } + }, + { + "id": "color", + "value": { + "mode": "continuous-GrYlRd" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Energy (kJ, window)" + }, + "properties": [ + { + "id": "decimals", + "value": 1 + }, + { + "id": "unit", + "value": "kJ" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge", + "valueDisplayMode": "text" + } + }, + { + "id": "color", + "value": { + "mode": "continuous-BlPu" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 8, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": [ + "Power (W)", + "Activity (F/s)", + "Energy (kJ)" + ], + "reducer": [ + "sum" + ], + "show": true + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Power (W)" + } + ] + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (pod, namespace) (rate({__name__=\"perf_core_energy_joules_total\", resctrl_group_source=\"pod\"}[$__rate_interval]) * on(k8s_pod_uid) group_left(pod, namespace) label_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{namespace=~\"$namespace\"}[$__range])), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\"))", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "power", + "interval": "5s" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (pod, namespace) (rate({__name__=\"perf_activity_farads_total\", resctrl_group_source=\"pod\"}[$__rate_interval]) * on(k8s_pod_uid) group_left(pod, namespace) label_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{namespace=~\"$namespace\"}[$__range])), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\"))", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "activity", + "interval": "5s" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (pod, namespace) (increase({__name__=\"perf_core_energy_joules_total\", resctrl_group_source=\"pod\"}[$__range]) * on(k8s_pod_uid) group_left(pod, namespace) label_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{namespace=~\"$namespace\"}[$__range])), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\")) / 1000", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "energy" + } + ], + "title": "Pod Energy & Activity Table", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "renameByName": { + "Value #activity": "Activity (F/s)", + "Value #energy": "Energy (kJ, window)", + "Value #power": "Power (W)", + "namespace": "Namespace", + "pod": "Pod" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 104, + "title": "Package Totals", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total power per CPU package (domain) across monitored pods (root/system counters are not observed)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Power (W)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (domain_id) (rate({__name__=\"perf_core_energy_joules_total\", resctrl_group_source=\"pod\"}[$__rate_interval]))", + "legendFormat": "{{domain_id}}", + "refId": "A", + "interval": "5s" + } + ], + "title": "Package Power (monitored pods)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total activity per package (domain) in F/s \u2014 across monitored pods (root/system counters are not observed)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Activity (F/s)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": " F/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (domain_id) (rate({__name__=\"perf_activity_farads_total\", resctrl_group_source=\"pod\"}[$__rate_interval]))", + "legendFormat": "{{domain_id}}", + "refId": "A", + "interval": "5s" + } + ], + "title": "Package Activity (monitored pods)", + "type": "timeseries" + } + ], + "refresh": "15s", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "resctrl", + "aet", + "energy", + "kubernetes" + ], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Prometheus", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(kube_pod_info, namespace)", + "hide": 0, + "includeAll": true, + "label": "Namespace", + "multi": true, + "name": "namespace", + "query": { + "query": "label_values(kube_pod_info, namespace)", + "refId": "ns" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Kubernetes Pod Energy (Intel AET / resctrl-mon)", + "uid": "resctrl-pod-energy" +} \ No newline at end of file diff --git a/deployment/helm/resctrl-mon/optional/otel-collector-agent.yaml b/deployment/helm/resctrl-mon/optional/otel-collector-agent.yaml new file mode 100644 index 000000000..f9331e68c --- /dev/null +++ b/deployment/helm/resctrl-mon/optional/otel-collector-agent.yaml @@ -0,0 +1,126 @@ +# Optional OTel Collector agent DaemonSet for resctrl-mon OTLP push path. +# +# Deploy this alongside nri-resctrl-mon when telemetry.otlp.enabled=true. +# It receives OTLP from the plugin, enriches with k8sattributes, and +# fans out to Prometheus and/or other backends. +# +# Prerequisites: +# - otel-collector-rbac.yaml (ServiceAccount + ClusterRole for pod metadata) +# +# Usage: +# kubectl apply -f otel-collector-rbac.yaml +# kubectl apply -f otel-collector-agent.yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: otel-collector-resctrl-config + namespace: monitoring +data: + config.yaml: | + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + processors: + # resctrl-mon emits k8s.pod.uid as a data-point attribute. The + # k8sattributes pod_association can only match on resource attributes, + # so promote it (and any other shared data-point attributes) to the + # resource level first with groupbyattrs. + groupbyattrs: + keys: + - k8s.pod.uid + k8sattributes: + auth_type: serviceAccount + extract: + metadata: + - k8s.namespace.name + - k8s.pod.name + - k8s.node.name + pod_association: + - sources: + - from: resource_attribute + name: k8s.pod.uid + batch: + timeout: 10s + exporters: + prometheus: + endpoint: 0.0.0.0:8889 + resource_to_telemetry_conversion: + enabled: true + service: + pipelines: + metrics: + receivers: [otlp] + processors: [groupbyattrs, k8sattributes, batch] + exporters: [prometheus] +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: otel-collector-resctrl + namespace: monitoring + labels: + app.kubernetes.io/name: otel-collector-resctrl +spec: + selector: + matchLabels: + app.kubernetes.io/name: otel-collector-resctrl + template: + metadata: + labels: + app.kubernetes.io/name: otel-collector-resctrl + spec: + serviceAccountName: otel-collector-resctrl + containers: + - name: collector + image: otel/opentelemetry-collector-contrib:0.104.0 + args: ["--config=/etc/otel/config.yaml"] + ports: + - name: otlp-grpc + containerPort: 4317 + protocol: TCP + - name: prom-export + containerPort: 8889 + protocol: TCP + volumeMounts: + - name: config + mountPath: /etc/otel + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + memory: 256Mi + volumes: + - name: config + configMap: + name: otel-collector-resctrl-config +--- +# Service exposing the OTLP gRPC receiver so the plugin can reach the +# collector at otel-collector-resctrl.monitoring.svc:4317. Because the +# collector runs as a DaemonSet, this ClusterIP Service load-balances across +# all node-local collectors; enrichment is correct regardless of which pod +# handles a request (k8sattributes has cluster-wide pod visibility). For +# strict node-local delivery, have the plugin target the node's host IP +# (status.hostIP) instead. +apiVersion: v1 +kind: Service +metadata: + name: otel-collector-resctrl + namespace: monitoring + labels: + app.kubernetes.io/name: otel-collector-resctrl +spec: + selector: + app.kubernetes.io/name: otel-collector-resctrl + ports: + - name: otlp-grpc + port: 4317 + targetPort: otlp-grpc + protocol: TCP + - name: prom-export + port: 8889 + targetPort: prom-export + protocol: TCP diff --git a/deployment/helm/resctrl-mon/optional/otel-collector-rbac.yaml b/deployment/helm/resctrl-mon/optional/otel-collector-rbac.yaml new file mode 100644 index 000000000..672693451 --- /dev/null +++ b/deployment/helm/resctrl-mon/optional/otel-collector-rbac.yaml @@ -0,0 +1,41 @@ +# RBAC for the OTel Collector k8sattributes processor. +# Grants read access to pod metadata so the processor can enrich metrics. +--- +# The reference manifests deploy into the monitoring namespace. Create it here +# so a clean cluster can `kubectl apply` these files without a prior step; +# apply is idempotent if the namespace already exists. +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: otel-collector-resctrl + namespace: monitoring +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: otel-collector-resctrl +rules: + - apiGroups: [""] + resources: ["pods", "namespaces", "nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: ["replicasets"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: otel-collector-resctrl +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: otel-collector-resctrl +subjects: + - kind: ServiceAccount + name: otel-collector-resctrl + namespace: monitoring diff --git a/deployment/helm/resctrl-mon/templates/configmap.yaml b/deployment/helm/resctrl-mon/templates/configmap.yaml index 75562a9bf..c64d426c9 100644 --- a/deployment/helm/resctrl-mon/templates/configmap.yaml +++ b/deployment/helm/resctrl-mon/templates/configmap.yaml @@ -10,3 +10,28 @@ data: resctrlPath: {{ .Values.resctrlPath }} namespaces: [] labelSelector: {} + telemetry: + prometheus: + enabled: {{ .Values.telemetry.prometheus.enabled }} + listenAddress: {{ .Values.telemetry.prometheus.listenAddress | quote }} + namespace: {{ .Values.telemetry.prometheus.namespace | quote }} + otlp: + enabled: {{ .Values.telemetry.otlp.enabled }} + endpoint: {{ .Values.telemetry.otlp.endpoint | quote }} + protocol: {{ .Values.telemetry.otlp.protocol | quote }} + interval: {{ .Values.telemetry.otlp.interval | quote }} + insecure: {{ .Values.telemetry.otlp.insecure }} + perfCounters: + enabled: {{ .Values.telemetry.perfCounters.enabled }} + {{- with .Values.telemetry.perfCounters.include }} + include: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.telemetry.perfCounters.exclude }} + exclude: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.telemetry.resourceAttributes }} + resourceAttributes: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/deployment/helm/resctrl-mon/templates/daemonset.yaml b/deployment/helm/resctrl-mon/templates/daemonset.yaml index 629a52dda..9293451d9 100644 --- a/deployment/helm/resctrl-mon/templates/daemonset.yaml +++ b/deployment/helm/resctrl-mon/templates/daemonset.yaml @@ -13,6 +13,13 @@ spec: metadata: labels: {{- include "nri-plugin.labels" . | nindent 8 }} + {{- if .Values.telemetry.prometheus.enabled }} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "{{ (split ":" .Values.telemetry.prometheus.listenAddress)._1 | default "9100" }}" + prometheus.io/path: "/metrics" + prometheus.io/interval: "{{ .Values.telemetry.prometheus.scrapeInterval }}" + {{- end }} spec: {{- with .Values.tolerations }} tolerations: @@ -61,6 +68,12 @@ spec: - -v image: {{ .Values.image.name }}:{{ .Values.image.tag | default .Chart.AppVersion }} imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.telemetry.prometheus.enabled }} + ports: + - name: metrics + containerPort: {{ (split ":" .Values.telemetry.prometheus.listenAddress)._1 | default "9100" | int }} + protocol: TCP + {{- end }} resources: requests: cpu: {{ .Values.resources.cpu }} diff --git a/deployment/helm/resctrl-mon/values.yaml b/deployment/helm/resctrl-mon/values.yaml index 26abf0cd0..68bc01cbe 100644 --- a/deployment/helm/resctrl-mon/values.yaml +++ b/deployment/helm/resctrl-mon/values.yaml @@ -4,6 +4,25 @@ --- resctrlPath: /sys/fs/resctrl +# Telemetry configuration — embedded OTel exporter. +telemetry: + prometheus: + enabled: true + listenAddress: ":9100" + scrapeInterval: "15s" # recommended scrape interval (annotation hint) + namespace: "" # empty = no prefix (l3_*/perf_* metric names) + otlp: + enabled: false + endpoint: "" # e.g. "otel-collector-resctrl.monitoring.svc:4317" + protocol: grpc # grpc | http + interval: 15s + insecure: true + perfCounters: + enabled: false # gate rdt=perf counters (c1_res, stalls_*, etc.) + include: [] + exclude: [] + resourceAttributes: {} # static OTel resource attributes + image: name: ghcr.io/containers/nri-plugins/nri-resctrl-mon # tag, if defined will use the given image tag, otherwise Chart.AppVersion will be used diff --git a/go.mod b/go.mod index eebdbe85b..067f21b6d 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,6 @@ require ( github.com/containers/nri-plugins/pkg/topology v0.0.0 github.com/coreos/go-systemd/v22 v22.7.0 github.com/fsnotify/fsnotify v1.10.1 - github.com/google/uuid v1.6.0 github.com/intel/goresctrl v0.13.0 github.com/intel/memtierd v0.1.1 github.com/k8stopologyawareschedwg/noderesourcetopology-api v0.1.3 @@ -67,6 +66,7 @@ require ( github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -114,6 +114,9 @@ require ( replace ( github.com/containers/nri-plugins/pkg/topology v0.0.0 => ./pkg/topology + // Temporary: depends on unreleased goresctrl pkg/monitor (intel/goresctrl#192). + // Drop once a goresctrl release containing pkg/monitor is available. + github.com/intel/goresctrl v0.13.0 => github.com/cmcantalupo/goresctrl v0.0.0-20260812225045-3705888589be github.com/opencontainers/runtime-tools => github.com/opencontainers/runtime-tools v0.0.0-20221026201742-946c877fa809 ) diff --git a/go.sum b/go.sum index 19e7836e8..985dfef06 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1x github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cmcantalupo/goresctrl v0.0.0-20260812225045-3705888589be h1:/U5tRh81tg8WRT1eumPNo7AUP/p0PT3MsXEhRunRssI= +github.com/cmcantalupo/goresctrl v0.0.0-20260812225045-3705888589be/go.mod h1:tCyfuJ95wo5HR0SI2TybDiew+BN5wocABcaX8N7ZAQg= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/nri v0.12.2 h1:piA7h2QUm0m3+e994qWFPYwnoXXwAphwDc1Ydx1eWi4= @@ -71,8 +73,6 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= -github.com/intel/goresctrl v0.13.0 h1:5fhKjNq4V5MYDFHa//6M6x0jP6Iq5EXwZc6/eYxdEtQ= -github.com/intel/goresctrl v0.13.0/go.mod h1:KFHS91JGOmeeuEog+nTQcsGjLC81nRqdsdhcqf69fjU= github.com/intel/memtierd v0.1.1 h1:hGSN0+dzjaUkwgkJrk6B9SU4dntggXLpXgs9Dm+jfz4= github.com/intel/memtierd v0.1.1/go.mod h1:NFDBvjoDS42gBK/c9q/CYCJ2pt/+g7UQwOOBvQli4z0= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= diff --git a/sample-configs/nri-resctrl-mon.yaml b/sample-configs/nri-resctrl-mon.yaml index 9cbd339a0..a3b649e9e 100644 --- a/sample-configs/nri-resctrl-mon.yaml +++ b/sample-configs/nri-resctrl-mon.yaml @@ -1,3 +1,19 @@ resctrlPath: /sys/fs/resctrl namespaces: [] labelSelector: {} +telemetry: + prometheus: + enabled: true + listenAddress: ":9100" + namespace: "" + otlp: + enabled: false + endpoint: "" + protocol: grpc + interval: 15s + insecure: true + perfCounters: + enabled: false + include: [] + exclude: [] + resourceAttributes: {}