From 87e5c49413dfc3ce8699ef6525a1ae5a11bea12a Mon Sep 17 00:00:00 2001 From: "Christopher M. Cantalupo" Date: Fri, 21 Aug 2026 11:50:54 -0700 Subject: [PATCH 1/2] resctrl-mon: OTel telemetry via goresctrl pkg/monitor Rework the resctrl-mon NRI plugin to manage per-pod resctrl monitoring groups and export their counters through goresctrl/pkg/monitor and an embedded OpenTelemetry SDK, replacing the plugin's inline resctrl handling. Depends on goresctrl PR intel/goresctrl#192 (pkg/monitor), which is not yet merged or released. Until a goresctrl release containing pkg/monitor is available, go.mod carries a temporary replace pointing github.com/intel/ goresctrl at the PR #192 head commit (cmcantalupo/goresctrl@3705888). That replace must be dropped and the require bumped once the release ships; the plugin is not mergeable until then. Plugin: - Replace inline resctrl management (resctrl.go, state.go) with monitor.Manager; delegate instrument naming, monotonic accumulation, and counter discovery to monitor.RegisterOTelInstruments(). - Embed an OTel SDK (telemetry.go): Prometheus pull exporter on :9100 plus optional OTLP push. Tear down the OTel registration on onClose and on dynamic config reload so the previous Manager/registration is not leaked and telemetry is rebound to the new Manager. - perfCounterFilter gates perf-counter include/exclude; groupAttributes injects k8s.pod.uid, resctrl.control_group, source. Float64 counter fidelity is preserved from the kernel (no integer truncation). - mon_group lifetime is scoped to the pod sandbox: PostCreateContainer creates the group idempotently and RemovePodSandbox tears it down. StopContainer is intentionally not handled, so a container restart keeps the RMID stable and avoids residual-counter energy spikes; orphans from a missed teardown are reconciled in the background. Metric names use the domain-derived convention, aligning L3 counters with pkg/rdt's existing Prometheus names: l3_llc_occupancy_bytes, l3_mbm_local_bytes_total, l3_mbm_total_bytes_total, perf_core_energy_joules_total, perf_activity_farads_total Helm chart: - values.yaml telemetry block (prometheus/otlp/perfCounters); ConfigMap renders the full telemetry config; container port 9100 with prometheus.io/scrape annotations. - Optional OTel Collector DaemonSet + k8sattributes RBAC manifests. - Two Grafana dashboards using the OTel metric names, joined to kube_pod_info on the Pod UID. The join collapses kube_pod_info to one series per Pod before the many-to-one match, so a freshly created Pod (whose empty pod_ip yields a second series for the same uid) no longer fails the query with a duplicate-match-group error. Makefile: add an install-plugins target for static NRI discovery. Signed-off-by: Jedrzej Wasiukiewicz Signed-off-by: Christopher M. Cantalupo --- Makefile | 20 + cmd/plugins/resctrl-mon/main.go | 5 + cmd/plugins/resctrl-mon/metrics.go | 121 ++ cmd/plugins/resctrl-mon/plugin.go | 260 ++-- cmd/plugins/resctrl-mon/plugin_test.go | 328 ++--- cmd/plugins/resctrl-mon/resctrl.go | 194 --- cmd/plugins/resctrl-mon/resctrl_test.go | 251 ---- cmd/plugins/resctrl-mon/state.go | 113 -- cmd/plugins/resctrl-mon/telemetry.go | 243 ++++ cmd/plugins/resctrl-mon/telemetry_test.go | 336 +++++ deployment/helm/resctrl-mon/README.md | 63 + .../grafana-resctrl-perf-counters.json | 996 +++++++++++++++ .../optional/grafana-resctrl-pod-energy.json | 1088 +++++++++++++++++ .../optional/otel-collector-agent.yaml | 92 ++ .../optional/otel-collector-rbac.yaml | 33 + .../helm/resctrl-mon/templates/configmap.yaml | 25 + .../helm/resctrl-mon/templates/daemonset.yaml | 13 + deployment/helm/resctrl-mon/values.yaml | 19 + go.mod | 5 +- go.sum | 4 +- sample-configs/nri-resctrl-mon.yaml | 16 + 21 files changed, 3306 insertions(+), 919 deletions(-) create mode 100644 cmd/plugins/resctrl-mon/metrics.go delete mode 100644 cmd/plugins/resctrl-mon/resctrl.go delete mode 100644 cmd/plugins/resctrl-mon/resctrl_test.go delete mode 100644 cmd/plugins/resctrl-mon/state.go create mode 100644 cmd/plugins/resctrl-mon/telemetry.go create mode 100644 cmd/plugins/resctrl-mon/telemetry_test.go create mode 100644 deployment/helm/resctrl-mon/optional/grafana-resctrl-perf-counters.json create mode 100644 deployment/helm/resctrl-mon/optional/grafana-resctrl-pod-energy.json create mode 100644 deployment/helm/resctrl-mon/optional/otel-collector-agent.yaml create mode 100644 deployment/helm/resctrl-mon/optional/otel-collector-rbac.yaml diff --git a/Makefile b/Makefile index 1ccdefbcf..dbafb92ba 100644 --- a/Makefile +++ b/Makefile @@ -154,6 +154,26 @@ test: test-gopkgs verify: verify-godeps verify-fmt verify-generate verify-build verify-docs +# +# install targets +# + +# NRI plugin install directory and default index for static plugin discovery. +NRI_PLUGINS_DIR ?= /opt/nri/plugins +NRI_DEFAULT_IDX ?= 90 + +# install-plugins: install plugin binaries to NRI_PLUGINS_DIR with the +# - prefix required by CRI-O/containerd static plugin discovery. +install-plugins: build-plugins + $(Q)echo "Installing plugins to $(NRI_PLUGINS_DIR)..."; \ + mkdir -p $(NRI_PLUGINS_DIR); \ + for bin in $(PLUGINS); do \ + name=$${bin#nri-}; \ + dst="$(NRI_PLUGINS_DIR)/$(NRI_DEFAULT_IDX)-$$name"; \ + install -m 755 "$(BIN_PATH)/$$bin" "$$dst"; \ + echo " $$dst"; \ + done + # # build targets # diff --git a/cmd/plugins/resctrl-mon/main.go b/cmd/plugins/resctrl-mon/main.go index eedf56599..fc6eb5f85 100644 --- a/cmd/plugins/resctrl-mon/main.go +++ b/cmd/plugins/resctrl-mon/main.go @@ -69,6 +69,11 @@ func main() { } } + // Start OTel telemetry (Prometheus endpoint + optional OTLP push). + if err := p.startTelemetry(context.Background()); err != nil { + log.Fatalf("failed to start telemetry: %v", err) + } + opts := []stub.Option{ stub.WithOnClose(p.onClose), } diff --git a/cmd/plugins/resctrl-mon/metrics.go b/cmd/plugins/resctrl-mon/metrics.go new file mode 100644 index 000000000..d44ba4ed3 --- /dev/null +++ b/cmd/plugins/resctrl-mon/metrics.go @@ -0,0 +1,121 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "path/filepath" + "regexp" + "strings" + + "github.com/intel/goresctrl/pkg/monitor" + "go.opentelemetry.io/otel/attribute" + otelmetric "go.opentelemetry.io/otel/metric" +) + +// coreAETFiles are counter files that appear under mon_PERF_PKG_* but are +// always exported (not gated by perfCounters.enabled). +var coreAETFiles = map[string]bool{ + "core_energy": true, + "activity": true, +} + +// setupMetrics registers OTel instruments via the goresctrl adapter. +func setupMetrics(mgr *monitor.Manager, cfg telemetryConfig, meter otelmetric.Meter) (*monitor.Registration, error) { + return mgr.RegisterOTelInstruments(meter, + monitor.WithFilter(perfCounterFilter(cfg)), + monitor.WithAttributes(groupAttributes), + ) +} + +// perfCounterFilter returns a FilterFunc that implements the perf counter gate. +func perfCounterFilter(cfg telemetryConfig) monitor.FilterFunc { + return func(r monitor.Reading) bool { + // Core AET files (core_energy, activity) are always allowed regardless + // of which domain they appear under. + if coreAETFiles[r.Name] { + return true + } + // Non-PERF_PKG domains (mon_L3_*) are always allowed. + if !strings.HasPrefix(r.Domain, "mon_PERF_PKG_") { + return true + } + // This is a perf counter under mon_PERF_PKG_*. Check the gate. + if !cfg.PerfCounters.Enabled { + return false + } + // Apply include/exclude lists if configured. + if len(cfg.PerfCounters.Include) > 0 { + for _, pattern := range cfg.PerfCounters.Include { + if matchGlob(pattern, r.Name) { + return true + } + } + return false + } + if len(cfg.PerfCounters.Exclude) > 0 { + for _, pattern := range cfg.PerfCounters.Exclude { + if matchGlob(pattern, r.Name) { + return false + } + } + } + return true + } +} + +// groupAttributes provides per-group OTel attributes. +func groupAttributes(key, path string) []attribute.KeyValue { + return []attribute.KeyValue{ + attribute.String("k8s.pod.uid", key), + attribute.String("resctrl.control_group", controlGroupOf(path)), + attribute.String("resctrl.group.source", sourceFor(key)), + } +} + +// matchGlob does simple glob matching (only * is supported as wildcard). +func matchGlob(pattern, name string) bool { + if pattern == name { + return true + } + if strings.Contains(pattern, "*") { + re := "^" + strings.ReplaceAll(regexp.QuoteMeta(pattern), `\*`, ".*") + "$" + matched, _ := regexp.MatchString(re, name) + return matched + } + return false +} + +// controlGroupOf extracts the CTRL group name from a mon_group path. +// e.g. "/sys/fs/resctrl/COS1/mon_groups/abc-123" → "COS1" +// e.g. "/sys/fs/resctrl/mon_groups/abc-123" → "" (root/default) +func controlGroupOf(groupPath string) string { + ctrlDir := filepath.Dir(filepath.Dir(groupPath)) + base := filepath.Base(ctrlDir) + if base == "resctrl" || base == "." || base == "/" { + return "" + } + return base +} + +// uuidPattern matches standard dashed UUID (pod UIDs). +var uuidPattern = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) + +// sourceFor classifies a group key as "pod" (UUID) or "other". +func sourceFor(key string) string { + if uuidPattern.MatchString(key) { + return "pod" + } + return "other" +} diff --git a/cmd/plugins/resctrl-mon/plugin.go b/cmd/plugins/resctrl-mon/plugin.go index 30f5512a2..1828fd477 100644 --- a/cmd/plugins/resctrl-mon/plugin.go +++ b/cmd/plugins/resctrl-mon/plugin.go @@ -16,36 +16,40 @@ package main import ( "context" + "errors" "fmt" "os" "path/filepath" "slices" "strconv" "strings" - "sync" "time" - "github.com/google/uuid" "sigs.k8s.io/yaml" "github.com/containerd/nri/pkg/api" "github.com/containerd/nri/pkg/stub" + "github.com/intel/goresctrl/pkg/monitor" ) const ( // reconcileInterval is how often the background reconciler checks for // orphaned mon_groups left behind by failed StopContainer removals. reconcileInterval = 30 * time.Second + + // telemetryShutdownTimeout bounds how long onClose waits for the telemetry + // stack (MeterProvider flush + HTTP server) to drain before exiting. + telemetryShutdownTimeout = 5 * time.Second ) // plugin implements the NRI plugin interface for resctrl monitoring groups. type plugin struct { stub stub.Stub config *pluginConfig - state *podState - rdt *resctrlOps - mu sync.Mutex // serializes ensureMonGroup to prevent TOCTOU races + mgr *monitor.Manager stopReconciler chan struct{} // closed to stop the background reconciler + telemetry *telemetryState + metrics *monitor.Registration } // pluginConfig holds the runtime configuration for the plugin. @@ -60,16 +64,29 @@ type pluginConfig struct { // LabelSelector filters mon_group creation to pods matching these labels. // Empty map means all pods. LabelSelector map[string]string `json:"labelSelector"` + + // Telemetry configures the embedded OTel exporter (Prometheus + OTLP). + Telemetry telemetryConfig `json:"telemetry"` } +const defaultResctrlPath = "/sys/fs/resctrl" + func newPlugin() *plugin { cfg := &pluginConfig{ ResctrlPath: defaultResctrlPath, + Telemetry: defaultTelemetryConfig(), + } + mgr, err := monitor.New(monitor.Options{ + ResctrlRoot: cfg.ResctrlPath, + KeyValidator: monitor.PodUIDValidator, + KeyCanonicalizer: monitor.CanonicalizePodUID, + }) + if err != nil { + log.Fatalf("failed to create monitor manager: %v", err) } return &plugin{ config: cfg, - state: newPodState(), - rdt: newResctrlOps(cfg.ResctrlPath), + mgr: mgr, } } @@ -93,6 +110,15 @@ func (p *plugin) onClose() { if p.stopReconciler != nil { close(p.stopReconciler) } + if p.metrics != nil { + _ = p.metrics.Unregister() + p.metrics = nil + } + if p.telemetry != nil { + ctx, cancel := context.WithTimeout(context.Background(), telemetryShutdownTimeout) + p.telemetry.shutdown(ctx) + cancel() + } log.Infof("Connection to the runtime lost, exiting...") os.Exit(0) } @@ -102,6 +128,7 @@ func (p *plugin) setConfig(data []byte) error { log.Tracef("setConfig: parsing\n---8<---\n%s\n--->8---", data) cfg := pluginConfig{ ResctrlPath: defaultResctrlPath, + Telemetry: defaultTelemetryConfig(), } if err := yaml.Unmarshal(data, &cfg); err != nil { return fmt.Errorf("setConfig: cannot parse configuration: %w", err) @@ -111,8 +138,53 @@ func (p *plugin) setConfig(data []byte) error { return fmt.Errorf("setConfig: resctrlPath must be an absolute path, got %q", cfg.ResctrlPath) } cfg.ResctrlPath = resctrlPath + if err := validateTelemetryConfig(&cfg.Telemetry); err != nil { + return fmt.Errorf("setConfig: %w", err) + } + + // Recreate the monitor manager with the new path before mutating any plugin + // state, so a failure here leaves the running configuration fully intact. + mgr, err := monitor.New(monitor.Options{ + ResctrlRoot: cfg.ResctrlPath, + KeyValidator: monitor.PodUIDValidator, + KeyCanonicalizer: monitor.CanonicalizePodUID, + }) + if err != nil { + return fmt.Errorf("setConfig: failed to create monitor manager: %w", err) + } + p.config = &cfg - p.rdt = newResctrlOps(cfg.ResctrlPath) + + // Stop the old reconciler before swapping the manager; it holds a + // reference to the old manager and would otherwise leak. + if p.stopReconciler != nil { + close(p.stopReconciler) + p.stopReconciler = nil + } + + // If telemetry is already running, its OTel batch callback is bound to the + // manager we are about to replace (which is pinned to the previous + // ResctrlPath). Tear it down so we neither leak the old manager nor keep a + // second registration reading the old root, then restart it against the new + // manager below. + restartTelemetry := p.telemetry != nil + if restartTelemetry { + if p.metrics != nil { + _ = p.metrics.Unregister() + p.metrics = nil + } + p.telemetry.shutdown(context.Background()) + p.telemetry = nil + } + + p.mgr = mgr + + if restartTelemetry { + if err := p.startTelemetry(context.Background()); err != nil { + return fmt.Errorf("setConfig: restart telemetry: %w", err) + } + } + log.Debugf("configuration: resctrlPath=%s namespaces=%v labelSelector=%v", cfg.ResctrlPath, cfg.Namespaces, cfg.LabelSelector) return nil @@ -132,6 +204,7 @@ func (p *plugin) Synchronize(ctx context.Context, pods []*api.PodSandbox, contai // Create mon_groups for running containers that don't have one, // and write their PIDs to ensure monitoring is active after restart. + var liveKeys []string for _, ctr := range containers { pod, ok := podBySandboxID[ctr.GetPodSandboxId()] if !ok { @@ -143,32 +216,34 @@ func (p *plugin) Synchronize(ctx context.Context, pods []*api.PodSandbox, contai } podUID := pod.GetUid() rdtClass := getRDTClass(ctr) - if err := p.ensureMonGroup(podUID, ctr.GetId(), rdtClass); err != nil { + + grp, err := p.mgr.EnsureGroup(podUID, rdtClass) + if err != nil { log.Warnf("Synchronize: failed to create mon_group for pod %s: %v", podUID, err) continue } - // Use canonical form for state lookups (ensureMonGroup stores canonical). - u, _ := uuid.Parse(podUID) - canonicalUID := u.String() + liveKeys = append(liveKeys, podUID) + pid := int(ctr.GetPid()) if pid > 0 { - monGroupDir := p.state.getMonGroupDir(canonicalUID) - if err := p.rdt.writeTaskPID(monGroupDir, pid); err != nil { + if err := p.mgr.AssignPID(podUID, pid); err != nil { log.Warnf("Synchronize: failed to write PID %d for pod %s: %v", pid, podUID, err) } else { - log.Debugf("Synchronize: assigned pid %d for pod %s", pid, podUID) + log.Debugf("Synchronize: assigned pid %d for pod %s in %s", pid, podUID, grp.Path()) } } } // Remove orphaned mon_groups from a previous plugin instance. - p.rdt.cleanOrphanedMonGroups(p.state) + if err := p.mgr.Reconcile(liveKeys); err != nil { + log.Warnf("Synchronize: reconcile failed: %v", err) + } // Start the background reconciler to periodically clean up orphaned // mon_groups that could not be removed during StopContainer. p.startReconciler() - log.Infof("synchronization complete: tracking %d pods", p.state.podCount()) + log.Infof("synchronization complete: tracking %d pods", len(p.mgr.List())) return nil, nil } @@ -189,7 +264,9 @@ func (p *plugin) startReconciler() { case <-p.stopReconciler: return case <-ticker.C: - p.rdt.cleanOrphanedMonGroups(p.state) + if err := p.mgr.Reconcile(p.mgr.List()); err != nil { + log.Warnf("reconciler: %v", err) + } } } }() @@ -212,7 +289,7 @@ func (p *plugin) PostCreateContainer(ctx context.Context, pod *api.PodSandbox, c } rdtClass := getRDTClass(ctr) - if err := p.ensureMonGroup(podUID, ctr.GetId(), rdtClass); err != nil { + if _, err := p.mgr.EnsureGroup(podUID, rdtClass); err != nil { log.Warnf("PostCreateContainer %s: failed to create mon_group: %v", ctrName, err) return nil // non-fatal: don't block container creation } @@ -227,14 +304,8 @@ func (p *plugin) PostCreateContainer(ctx context.Context, pod *api.PodSandbox, c // This is the ideal moment to write the PID to the resctrl mon_group tasks // file: the kernel assigns the RMID to this PID, and when the process starts // and forks threads they all inherit the RMID automatically. -// -// If the PID is not available (should not happen at this stage), we fall back -// to PostStartContainer which will write PIDs after the process starts. func (p *plugin) StartContainer(ctx context.Context, pod *api.PodSandbox, ctr *api.Container) error { podUID := pod.GetUid() - if u, err := uuid.Parse(podUID); err == nil { - podUID = u.String() - } ctrName := pprintCtr(pod, ctr) pid := int(ctr.GetPid()) @@ -244,17 +315,11 @@ func (p *plugin) StartContainer(ctx context.Context, pod *api.PodSandbox, ctr *a return nil } - monGroupDir := p.state.getMonGroupDir(podUID) - if monGroupDir == "" { - log.Debugf("StartContainer %s: no mon_group (pod not tracked), skipping", ctrName) - return nil - } - if pid > 0 { - if err := p.rdt.writeTaskPID(monGroupDir, pid); err != nil { - log.Warnf("StartContainer %s: failed to write PID %d to tasks: %v", ctrName, pid, err) + if err := p.mgr.AssignPID(podUID, pid); err != nil { + log.Warnf("StartContainer %s: failed to assign PID %d: %v", ctrName, pid, err) } else { - log.Infof("StartContainer %s: assigned pid %d to mon_group %s (pre-start, no threads yet)", ctrName, pid, monGroupDir) + log.Infof("StartContainer %s: assigned pid %d (pre-start, no threads yet)", ctrName, pid) } } else { log.Warnf("StartContainer %s: PID not available at pre-start, will retry in PostStartContainer", ctrName) @@ -265,13 +330,9 @@ func (p *plugin) StartContainer(ctx context.Context, pod *api.PodSandbox, ctr *a // PostStartContainer is called after the container process has been started. // This is a fallback: if StartContainer did not have the PID, we write the -// init PID here. The init PID is sufficient because all child threads inherit -// the RMID. +// init PID here. func (p *plugin) PostStartContainer(ctx context.Context, pod *api.PodSandbox, ctr *api.Container) error { podUID := pod.GetUid() - if u, err := uuid.Parse(podUID); err == nil { - podUID = u.String() - } ctrName := pprintCtr(pod, ctr) pid := int(ctr.GetPid()) @@ -281,16 +342,11 @@ func (p *plugin) PostStartContainer(ctx context.Context, pod *api.PodSandbox, ct return nil } - monGroupDir := p.state.getMonGroupDir(podUID) - if monGroupDir == "" { - return nil - } - if pid > 0 { - if err := p.rdt.writeTaskPID(monGroupDir, pid); err != nil { - log.Warnf("PostStartContainer %s: failed to write PID %d to tasks: %v", ctrName, pid, err) + if err := p.mgr.AssignPID(podUID, pid); err != nil { + log.Warnf("PostStartContainer %s: failed to assign PID %d: %v", ctrName, pid, err) } else { - log.Infof("PostStartContainer %s: assigned pid %d to mon_group %s", ctrName, pid, monGroupDir) + log.Infof("PostStartContainer %s: assigned pid %d", ctrName, pid) } } else { log.Warnf("PostStartContainer %s: PID=0, cannot assign to mon_group (runtime did not provide PID via NRI)", ctrName) @@ -299,97 +355,37 @@ func (p *plugin) PostStartContainer(ctx context.Context, pod *api.PodSandbox, ct return nil } -// StopContainer is called when a container is being stopped. -func (p *plugin) StopContainer(ctx context.Context, pod *api.PodSandbox, ctr *api.Container) ([]*api.ContainerUpdate, error) { - podUID := pod.GetUid() - if u, err := uuid.Parse(podUID); err == nil { - podUID = u.String() - } - ctrName := pprintCtr(pod, ctr) - - log.Debugf("StopContainer %s", ctrName) - - monGroupDir := p.state.getMonGroupDir(podUID) - if monGroupDir == "" { - return nil, nil - } - - // Drop only the container from tracking. The mon_group is intentionally - // retained until the pod sandbox is removed (see RemovePodSandbox). - // - // A pod with restartPolicy Always/OnFailure restarts its container under - // the same pod UID after the process exits (e.g. a workload that runs for - // a fixed duration). If we removed the mon_group here, the kernel would - // release the RMID and reassign a fresh one on the next PostCreateContainer. - // The new RMID carries residual hardware counter values, producing a - // counter discontinuity that surfaces as a false energy/bandwidth spike in - // downstream rate() consumers. Tying mon_group lifetime to the pod sandbox - // keeps the RMID stable across container restarts. - p.state.removeContainer(podUID, ctr.GetId()) - - if p.state.podHasNoContainers(podUID) { - log.Debugf("StopContainer %s: last container stopped, retaining mon_group %s until pod removal", ctrName, monGroupDir) - } - - return nil, nil -} - -// RemovePodSandbox is called when a pod sandbox is torn down. This is the point -// at which the pod (and its UID) is truly gone, so it is the correct place to -// release the mon_group and its RMID. Removing the mon_group earlier (e.g. in -// StopContainer) would release the RMID across container restarts and cause -// false counter spikes; see StopContainer for details. +// StopContainer is intentionally not implemented. A container stop must NOT +// tear down the pod's mon_group: a restart keeps the pod sandbox alive, and +// releasing the RMID would give the replacement container a fresh RMID whose +// hardware counters carry a non-zeroed residual, producing a false energy +// spike. The mon_group is removed in RemovePodSandbox when the pod is truly +// gone (and the reconciler cleans orphans from any missed teardown events). +// Because the NRI stub derives its event subscription from the implemented +// handler interfaces, omitting StopContainer also unsubscribes the plugin from +// STOP_CONTAINER events entirely. + +// RemovePodSandbox is called when the pod sandbox is being torn down. +// This is the point at which the mon_group should be cleaned up, because +// the pod (and its UID) will not be reused. func (p *plugin) RemovePodSandbox(ctx context.Context, pod *api.PodSandbox) error { podUID := pod.GetUid() - if u, err := uuid.Parse(podUID); err == nil { - podUID = u.String() - } - monGroupDir := p.state.getMonGroupDir(podUID) - if monGroupDir == "" { - return nil - } - - log.Infof("RemovePodSandbox %s/%s: removing mon_group %s", pod.GetNamespace(), pod.GetName(), monGroupDir) - if err := p.rdt.removeMonGroup(monGroupDir); err != nil { + // Attempt removal unconditionally rather than gating on shouldMonitorPod: a + // pod may have been monitored under a configuration that was later changed to + // exclude it. Gating here would strand its mon_group, because Remove would + // never run and the key would linger in the Manager, so the reconciler would + // keep treating it as live and never reap it. Remove is idempotent and + // reports ErrNotTracked for a pod that was never monitored. + switch err := p.mgr.Remove(podUID); { + case err == nil: + log.Infof("RemovePodSandbox %s/%s: removed mon_group", pod.GetNamespace(), pod.GetName()) + case errors.Is(err, monitor.ErrNotTracked): + // Pod was never monitored; nothing to clean up. + default: log.Warnf("RemovePodSandbox %s/%s: failed to remove mon_group (will be cleaned by reconciler): %v", pod.GetNamespace(), pod.GetName(), err) } - p.state.removePod(podUID) - - return nil -} - -// ensureMonGroup creates the mon_group directory if it doesn't exist and registers -// the container in the in-memory state. -// -// Limitation: all containers in a pod share a single mon_group under the first -// container's RDT class. If an allocation plugin assigns different classes to -// containers in the same pod, subsequent containers use the first class. -func (p *plugin) ensureMonGroup(podUID, containerID, rdtClass string) error { - u, err := uuid.Parse(podUID) - if err != nil { - return fmt.Errorf("invalid pod UID %q", podUID) - } - podUID = u.String() - - p.mu.Lock() - defer p.mu.Unlock() - - if p.state.getMonGroupDir(podUID) != "" { - // Mon_group already exists for this pod. Just add the container. - p.state.addContainer(podUID, containerID) - return nil - } - - monGroupDir, err := p.rdt.createMonGroup(rdtClass, podUID) - if err != nil { - return err - } - - p.state.addPod(podUID, monGroupDir) - p.state.addContainer(podUID, containerID) - log.Infof("created mon_group %s for pod %s", monGroupDir, podUID) return nil } diff --git a/cmd/plugins/resctrl-mon/plugin_test.go b/cmd/plugins/resctrl-mon/plugin_test.go index a5d7215f1..da7479bc5 100644 --- a/cmd/plugins/resctrl-mon/plugin_test.go +++ b/cmd/plugins/resctrl-mon/plugin_test.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/containerd/nri/pkg/api" + "github.com/intel/goresctrl/pkg/monitor" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -35,10 +36,16 @@ func newTestPlugin(resctrlPath string) *plugin { cfg := &pluginConfig{ ResctrlPath: resctrlPath, } + mgr, err := monitor.New(monitor.Options{ + ResctrlRoot: resctrlPath, + KeyValidator: monitor.PodUIDValidator, + }) + if err != nil { + panic(err) + } return &plugin{ config: cfg, - state: newPodState(), - rdt: newResctrlOps(resctrlPath), + mgr: mgr, } } @@ -129,23 +136,27 @@ func TestPostCreateContainer_FilteredPod(t *testing.T) { require.NoError(t, err) // Pod should not be tracked since it's not in the production namespace. - assert.Equal(t, 0, p.state.podCount()) + assert.Equal(t, 0, len(p.mgr.List())) } func TestPostCreateContainer_CreatesMonGroup(t *testing.T) { tmpDir := t.TempDir() p := newTestPlugin(tmpDir) - pod := makePod("a1b2c3d4-e5f6-7890-abcd-ef1234567890", "default", "test-pod") - ctr := makeContainer("c1", "container1", "a1b2c3d4-e5f6-7890-abcd-ef1234567890", 0, "") + podUID := "a1b2c3d4-e5f6-7890-abcd-ef1234567890" + pod := makePod(podUID, "default", "test-pod") + ctr := makeContainer("c1", "container1", podUID, 0, "") err := p.PostCreateContainer(context.Background(), pod, ctr) require.NoError(t, err) // Pod should be tracked. - assert.Equal(t, 1, p.state.podCount()) - monDir := p.state.getMonGroupDir("a1b2c3d4-e5f6-7890-abcd-ef1234567890") - assert.Contains(t, monDir, "mon_groups/a1b2c3d4-e5f6-7890-abcd-ef1234567890") + assert.Equal(t, 1, len(p.mgr.List())) + + // Mon_group directory should exist, keyed by bare pod UID. + monDir := filepath.Join(tmpDir, "mon_groups", podUID) + _, err = os.Stat(monDir) + assert.NoError(t, err) } func TestPostCreateContainer_WithRDTClass(t *testing.T) { @@ -153,14 +164,17 @@ func TestPostCreateContainer_WithRDTClass(t *testing.T) { p := newTestPlugin(tmpDir) require.NoError(t, os.Mkdir(filepath.Join(tmpDir, "BestEffort"), 0755)) - pod := makePod("a1b2c3d4-e5f6-7890-abcd-ef1234567890", "default", "test-pod") - ctr := makeContainer("c1", "container1", "a1b2c3d4-e5f6-7890-abcd-ef1234567890", 0, "BestEffort") + podUID := "a1b2c3d4-e5f6-7890-abcd-ef1234567890" + pod := makePod(podUID, "default", "test-pod") + ctr := makeContainer("c1", "container1", podUID, 0, "BestEffort") err := p.PostCreateContainer(context.Background(), pod, ctr) require.NoError(t, err) - monDir := p.state.getMonGroupDir("a1b2c3d4-e5f6-7890-abcd-ef1234567890") - assert.Contains(t, monDir, "BestEffort/mon_groups/a1b2c3d4-e5f6-7890-abcd-ef1234567890") + // Mon_group should be under the ctrl_group. + monDir := filepath.Join(tmpDir, "BestEffort", "mon_groups", podUID) + _, err = os.Stat(monDir) + assert.NoError(t, err) } func TestMultiContainerPod(t *testing.T) { @@ -175,120 +189,18 @@ func TestMultiContainerPod(t *testing.T) { // First container creates the mon_group. err := p.PostCreateContainer(context.Background(), pod, ctr1) require.NoError(t, err) - assert.Equal(t, 1, p.state.podCount()) + assert.Equal(t, 1, len(p.mgr.List())) // Second container reuses the same mon_group. err = p.PostCreateContainer(context.Background(), pod, ctr2) require.NoError(t, err) - assert.Equal(t, 1, p.state.podCount()) // still one pod - - // Stopping first container should not remove the mon_group. - _, err = p.StopContainer(context.Background(), pod, ctr1) - require.NoError(t, err) - assert.Equal(t, 1, p.state.podCount()) - assert.False(t, p.state.podHasNoContainers(podUID)) + assert.Equal(t, 1, len(p.mgr.List())) // still one pod - // Stopping the last container retains the mon_group (it is released only - // when the pod sandbox is removed, so the RMID stays stable across - // container restarts). - _, err = p.StopContainer(context.Background(), pod, ctr2) - require.NoError(t, err) - assert.Equal(t, 1, p.state.podCount()) - assert.True(t, p.state.podHasNoContainers(podUID)) - - // Removing the pod sandbox releases the mon_group. + // RemovePodSandbox is what actually removes the mon_group; container + // stops do not affect it (the plugin no longer handles StopContainer). err = p.RemovePodSandbox(context.Background(), pod) require.NoError(t, err) - assert.Equal(t, 0, p.state.podCount()) -} - -func TestStopContainer_UnknownPod(t *testing.T) { - p := newTestPlugin(t.TempDir()) - - pod := makePod("unknown-uid", "default", "unknown-pod") - ctr := makeContainer("c1", "container1", "unknown-uid", 1234, "") - - updates, err := p.StopContainer(context.Background(), pod, ctr) - require.NoError(t, err) - assert.Nil(t, updates) -} - -// TestContainerRestart_RetainsMonGroup verifies that a container restart under -// the same pod UID (e.g. restartPolicy: Always after a fixed-duration workload -// exits) keeps the same mon_group directory, so the kernel does not release and -// reassign the RMID. RMID reassignment would carry residual hardware counter -// values and surface as a false counter spike. -func TestContainerRestart_RetainsMonGroup(t *testing.T) { - tmpDir := t.TempDir() - p := newTestPlugin(tmpDir) - podUID := "a1b2c3d4-e5f6-7890-abcd-ef1234567890" - - pod := makePod(podUID, "default", "restart-pod") - ctr := makeContainer("c1", "container1", podUID, 0, "") - - // Container starts: mon_group is created. - err := p.PostCreateContainer(context.Background(), pod, ctr) - require.NoError(t, err) - monDir := p.state.getMonGroupDir(podUID) - require.NotEmpty(t, monDir) - require.DirExists(t, monDir) - - // Container exits (workload timed out). The mon_group must be retained. - _, err = p.StopContainer(context.Background(), pod, ctr) - require.NoError(t, err) - assert.Equal(t, 1, p.state.podCount()) - assert.DirExists(t, monDir, "mon_group must survive a container restart") - - // kubelet restarts the container under the same pod UID. The same - // mon_group directory (and thus RMID) must be reused. - err = p.PostCreateContainer(context.Background(), pod, ctr) - require.NoError(t, err) - assert.Equal(t, monDir, p.state.getMonGroupDir(podUID), "restart must reuse the same mon_group") - - // Pod is finally deleted: mon_group is released. - err = p.RemovePodSandbox(context.Background(), pod) - require.NoError(t, err) - assert.Equal(t, 0, p.state.podCount()) - assert.NoDirExists(t, monDir) -} - -// TestRemovePodSandbox_UnknownPod verifies that removing a pod we never tracked -// is a no-op and does not error. -func TestRemovePodSandbox_UnknownPod(t *testing.T) { - p := newTestPlugin(t.TempDir()) - - pod := makePod("a1b2c3d4-e5f6-7890-abcd-ef1234567890", "default", "unknown-pod") - - err := p.RemovePodSandbox(context.Background(), pod) - require.NoError(t, err) - assert.Equal(t, 0, p.state.podCount()) -} - -// TestMissingPodUID_NoMonGroup verifies that a sandbox without a valid pod UID -// is handled as a safe no-op across the full lifecycle: no mon_group is created -// (mon_groups are keyed by pod UID, not per container), and the stop/remove -// handlers neither panic nor leak state. This documents that the plugin does -// not currently fall back to per-container monitoring when the UID is absent. -func TestMissingPodUID_NoMonGroup(t *testing.T) { - p := newTestPlugin(t.TempDir()) - - pod := makePod("", "default", "no-uid-pod") - ctr := makeContainer("c1", "container1", "", 1234, "") - - // Creation must not create a group and must be non-fatal. - require.NoError(t, p.PostCreateContainer(context.Background(), pod, ctr)) - assert.Equal(t, 0, p.state.podCount()) - - // The remaining handlers must be safe no-ops. - require.NoError(t, p.StartContainer(context.Background(), pod, ctr)) - require.NoError(t, p.PostStartContainer(context.Background(), pod, ctr)) - - _, err := p.StopContainer(context.Background(), pod, ctr) - require.NoError(t, err) - assert.Equal(t, 0, p.state.podCount()) - - require.NoError(t, p.RemovePodSandbox(context.Background(), pod)) - assert.Equal(t, 0, p.state.podCount()) + assert.Equal(t, 0, len(p.mgr.List())) } func TestSetConfig(t *testing.T) { @@ -338,25 +250,27 @@ func TestSynchronize_UsesUIDNotSandboxID(t *testing.T) { require.NoError(t, err) // The mon_group should be keyed by the K8s pod UID, not the sandbox ID. - assert.Equal(t, 1, p.state.podCount()) - assert.True(t, p.state.hasPod(podUID)) - assert.False(t, p.state.hasPod(pod.GetId())) - - monDir := p.state.getMonGroupDir(podUID) - assert.Contains(t, monDir, podUID) + tracked := p.mgr.List() + assert.Equal(t, 1, len(tracked)) + assert.Contains(t, tracked, podUID) + + // Mon_group directory should exist. + monDir := filepath.Join(tmpDir, "mon_groups", podUID) + _, err = os.Stat(monDir) + assert.NoError(t, err) } -func TestEnsureMonGroup_InvalidUID(t *testing.T) { +func TestPostCreateContainer_InvalidUID(t *testing.T) { p := newTestPlugin(t.TempDir()) - err := p.ensureMonGroup("", "c1", "") - assert.Error(t, err) - assert.Contains(t, err.Error(), "invalid pod UID") + // Invalid UID (not a UUID) — EnsureGroup fails due to PodUIDValidator. + pod := makePod("not-a-uuid", "default", "bad-pod") + ctr := makeContainer("c1", "container1", "not-a-uuid", 0, "") - err = p.ensureMonGroup("not-a-uuid", "c1", "") - assert.Error(t, err) - - assert.Equal(t, 0, p.state.podCount()) + err := p.PostCreateContainer(context.Background(), pod, ctr) + // Non-fatal: returns nil but does not track. + require.NoError(t, err) + assert.Equal(t, 0, len(p.mgr.List())) } func TestStartContainer_AssignsPID(t *testing.T) { @@ -371,8 +285,8 @@ func TestStartContainer_AssignsPID(t *testing.T) { err := p.PostCreateContainer(context.Background(), pod, ctr) require.NoError(t, err) - monDir := p.state.getMonGroupDir(podUID) - require.NotEmpty(t, monDir) + monDir := filepath.Join(tmpDir, "mon_groups", podUID) + require.DirExists(t, monDir) // Simulate the kernel creating the tasks file. require.NoError(t, os.WriteFile(filepath.Join(monDir, "tasks"), nil, 0644)) @@ -399,8 +313,7 @@ func TestStartContainer_PIDZero_FallbackToPostStart(t *testing.T) { err := p.PostCreateContainer(context.Background(), pod, ctr) require.NoError(t, err) - monDir := p.state.getMonGroupDir(podUID) - require.NotEmpty(t, monDir) + monDir := filepath.Join(tmpDir, "mon_groups", podUID) require.NoError(t, os.WriteFile(filepath.Join(monDir, "tasks"), nil, 0644)) // StartContainer with PID 0 should not fail (just warns). @@ -429,82 +342,7 @@ func TestStartContainer_FilteredPod(t *testing.T) { require.NoError(t, err) } -func TestCompactUID_EnsureMonGroupStoresCanonical(t *testing.T) { - tmpDir := t.TempDir() - p := newTestPlugin(tmpDir) - - compactUID := "a1b2c3d4e5f678901234abcdef567890" - canonicalUID := "a1b2c3d4-e5f6-7890-1234-abcdef567890" - - pod := makePod(compactUID, "default", "test-pod") - ctr := makeContainer("c1", "container1", compactUID, 0, "") - - err := p.PostCreateContainer(context.Background(), pod, ctr) - require.NoError(t, err) - - // State must be keyed under the canonical dashed form. - assert.True(t, p.state.hasPod(canonicalUID)) - assert.False(t, p.state.hasPod(compactUID)) - - monDir := p.state.getMonGroupDir(canonicalUID) - assert.Contains(t, monDir, canonicalUID) -} - -func TestCompactUID_StartContainerFindsMonGroup(t *testing.T) { - tmpDir := t.TempDir() - p := newTestPlugin(tmpDir) - - compactUID := "a1b2c3d4e5f678901234abcdef567890" - canonicalUID := "a1b2c3d4-e5f6-7890-1234-abcdef567890" - - pod := makePod(compactUID, "default", "test-pod") - ctr := makeContainer("c1", "container1", compactUID, 0, "") - - // Create mon_group via compact UID. - err := p.PostCreateContainer(context.Background(), pod, ctr) - require.NoError(t, err) - - monDir := p.state.getMonGroupDir(canonicalUID) - require.NotEmpty(t, monDir) - require.NoError(t, os.WriteFile(filepath.Join(monDir, "tasks"), nil, 0644)) - - // StartContainer also using compact UID must find and write to the same mon_group. - ctrWithPid := makeContainer("c1", "container1", compactUID, 77, "") - err = p.StartContainer(context.Background(), pod, ctrWithPid) - require.NoError(t, err) - - data, err := os.ReadFile(filepath.Join(monDir, "tasks")) - require.NoError(t, err) - assert.Equal(t, "77\n", string(data)) -} - -func TestCompactUID_RemovePodSandboxCleansUp(t *testing.T) { - tmpDir := t.TempDir() - p := newTestPlugin(tmpDir) - - compactUID := "a1b2c3d4e5f678901234abcdef567890" - canonicalUID := "a1b2c3d4-e5f6-7890-1234-abcdef567890" - - pod := makePod(compactUID, "default", "test-pod") - ctr := makeContainer("c1", "container1", compactUID, 0, "") - - err := p.PostCreateContainer(context.Background(), pod, ctr) - require.NoError(t, err) - assert.Equal(t, 1, p.state.podCount()) - - // Stopping the last container retains the mon_group. - _, err = p.StopContainer(context.Background(), pod, ctr) - require.NoError(t, err) - assert.Equal(t, 1, p.state.podCount()) - - // Removing the pod sandbox (compact UID) must normalize and clean up. - err = p.RemovePodSandbox(context.Background(), pod) - require.NoError(t, err) - assert.Equal(t, 0, p.state.podCount()) - assert.False(t, p.state.hasPod(canonicalUID)) -} - -func TestRemovePodSandbox_RemovesStateOnRmdirFailure(t *testing.T) { +func TestRemovePodSandbox_RetainsGroupOnRmdirFailure(t *testing.T) { tmpDir := t.TempDir() p := newTestPlugin(tmpDir) podUID := "a1b2c3d4-e5f6-7890-abcd-ef1234567890" @@ -515,24 +353,19 @@ func TestRemovePodSandbox_RemovesStateOnRmdirFailure(t *testing.T) { // Create the mon_group. err := p.PostCreateContainer(context.Background(), pod, ctr) require.NoError(t, err) - assert.Equal(t, 1, p.state.podCount()) + assert.Equal(t, 1, len(p.mgr.List())) - monDir := p.state.getMonGroupDir(podUID) - require.NotEmpty(t, monDir) + monDir := filepath.Join(tmpDir, "mon_groups", podUID) + require.DirExists(t, monDir) - // Put a file inside the mon_group dir so os.Remove fails (dir not empty). + // Put a file inside the mon_group dir so os.Remove (rmdir) would fail. require.NoError(t, os.WriteFile(filepath.Join(monDir, "tasks"), nil, 0644)) - // Stopping the last container retains the mon_group. - _, err = p.StopContainer(context.Background(), pod, ctr) - require.NoError(t, err) - assert.Equal(t, 1, p.state.podCount()) - - // RemovePodSandbox should still drop the pod from state even if rmdir - // fails (the orphaned directory is reaped later by the reconciler). + // RemovePodSandbox attempts removal; with a non-empty dir rmdir fails, + // so the entry remains in the manager (reconciler will retry later). err = p.RemovePodSandbox(context.Background(), pod) - require.NoError(t, err) - assert.Equal(t, 0, p.state.podCount()) + require.NoError(t, err) // handler does not propagate the rmdir error + assert.Equal(t, 1, len(p.mgr.List()), "entry retained when rmdir fails; reconciler will clean") } func TestCheckRuntimeVersion(t *testing.T) { @@ -571,3 +404,46 @@ func TestCheckRuntimeVersion(t *testing.T) { }) } } + +// TestSetConfig_ReloadTearsDownTelemetry verifies that a dynamic setConfig +// reload, after telemetry has started, unregisters the OTel instruments bound +// to the old manager and starts fresh telemetry against the new manager, +// rather than leaking the old registration. +func TestSetConfig_ReloadTearsDownTelemetry(t *testing.T) { + groups := map[string]map[string]map[string]string{ + "11111111-1111-1111-1111-111111111111": { + "mon_L3_00": {"llc_occupancy": "4096"}, + }, + } + root1 := setupTestResctrl(t, groups) + root2 := setupTestResctrl(t, groups) + + p := newTestPlugin(root1) + // Disable Prometheus so telemetry starts without binding a port. + p.config.Telemetry = defaultTelemetryConfig() + p.config.Telemetry.Prometheus.Enabled = false + + require.NoError(t, p.startTelemetry(context.Background())) + oldReg := p.metrics + oldTelem := p.telemetry + require.NotNil(t, oldReg) + require.NotNil(t, oldTelem) + + // Dynamic reconfiguration to a new resctrl root, telemetry still port-less. + data := []byte("resctrlPath: " + root2 + "\ntelemetry:\n prometheus:\n enabled: false\n") + require.NoError(t, p.setConfig(data)) + t.Cleanup(func() { + if p.telemetry != nil { + p.telemetry.shutdown(context.Background()) + } + }) + + // Telemetry and its registration were replaced, not leaked. + require.NotNil(t, p.telemetry) + require.NotNil(t, p.metrics) + assert.NotSame(t, oldTelem, p.telemetry) + assert.NotSame(t, oldReg, p.metrics) + + // The old registration was already unregistered; a second call is a no-op. + assert.NoError(t, oldReg.Unregister()) +} diff --git a/cmd/plugins/resctrl-mon/resctrl.go b/cmd/plugins/resctrl-mon/resctrl.go deleted file mode 100644 index b4c9f9ae6..000000000 --- a/cmd/plugins/resctrl-mon/resctrl.go +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright The NRI Plugins Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "errors" - "fmt" - "os" - "path/filepath" - "strconv" - "strings" - "syscall" - - "github.com/google/uuid" -) - -const ( - defaultResctrlPath = "/sys/fs/resctrl" - monGroupsDir = "mon_groups" -) - -// resctrlOps handles filesystem operations on the resctrl mount. -type resctrlOps struct { - resctrlPath string -} - -func newResctrlOps(resctrlPath string) *resctrlOps { - return &resctrlOps{ - resctrlPath: resctrlPath, - } -} - -// createMonGroup creates a mon_group directory under the appropriate ctrl_group -// and returns the full path. If rdtClass is empty, the mon_group is created -// under the root resctrl directory. -// -// The kernel assigns an RMID to the new mon_group on mkdir. If no RMIDs are -// available, mkdir returns ENOSPC. -func (r *resctrlOps) createMonGroup(rdtClass, podUID string) (string, error) { - parentDir := r.resctrlPath - if rdtClass != "" { - if !isValidRDTClass(rdtClass) { - return "", fmt.Errorf("invalid RDT class name %q", rdtClass) - } - parentDir = filepath.Join(r.resctrlPath, rdtClass) - } - - // When an RDT class is specified, the ctrl_group must already exist - // (created by an allocation plugin). Do not create it implicitly — - // that would make an unintended ctrl_group in the resctrl filesystem. - if rdtClass != "" { - info, err := os.Stat(parentDir) - if err != nil { - return "", fmt.Errorf("ctrl_group %s does not exist: %w", parentDir, err) - } - if !info.IsDir() { - return "", fmt.Errorf("ctrl_group %s is not a directory", parentDir) - } - } - - monGroupsPath := filepath.Join(parentDir, monGroupsDir) - monGroupDir := filepath.Join(monGroupsPath, podUID) - - // Ensure the mon_groups/ directory exists. On a real resctrl mount - // this is always present. For testing, create it if needed. - if err := os.MkdirAll(monGroupsPath, 0755); err != nil { - return "", fmt.Errorf("mon_groups dir not available at %s: %w", monGroupsPath, err) - } - - // Use Mkdir (not MkdirAll) for the final mon_group directory to - // avoid accidentally creating a ctrl_group if rdtClass is wrong. - if err := os.Mkdir(monGroupDir, 0755); err != nil { - if errors.Is(err, os.ErrExist) { - return monGroupDir, nil - } - if errors.Is(err, syscall.ENOSPC) { - return "", fmt.Errorf("no RMIDs available for pod %s: %w", podUID, err) - } - return "", fmt.Errorf("failed to create mon_group %s: %w", monGroupDir, err) - } - - return monGroupDir, nil -} - -// removeMonGroup removes a mon_group directory. The kernel releases the RMID. -func (r *resctrlOps) removeMonGroup(monGroupDir string) error { - err := os.Remove(monGroupDir) - if err != nil && !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("failed to remove mon_group %s: %w", monGroupDir, err) - } - return nil -} - -// writeTaskPID writes a PID to the mon_group's tasks file. The kernel assigns -// this PID (and all future child processes) to the mon_group's RMID. -func (r *resctrlOps) writeTaskPID(monGroupDir string, pid int) error { - tasksFile := filepath.Join(monGroupDir, "tasks") - f, err := os.OpenFile(tasksFile, os.O_WRONLY, 0) - if err != nil { - return fmt.Errorf("failed to open %s for pid %d: %w", tasksFile, pid, err) - } - defer func() { _ = f.Close() }() - data := []byte(strconv.Itoa(pid) + "\n") - if _, err := f.Write(data); err != nil { - return fmt.Errorf("failed to write pid %d to %s: %w", pid, tasksFile, err) - } - return nil -} - -// cleanOrphanedMonGroups removes mon_group directories that are not tracked -// in the given state. This handles cleanup after a plugin crash/restart. -func (r *resctrlOps) cleanOrphanedMonGroups(state *podState) { - // Scan root-level mon_groups. - r.cleanOrphanedInDir(filepath.Join(r.resctrlPath, monGroupsDir), state) - - // Scan ctrl_group-level mon_groups. - entries, err := os.ReadDir(r.resctrlPath) - if err != nil { - log.Warnf("cleanOrphanedMonGroups: failed to read %s: %v", r.resctrlPath, err) - return - } - for _, entry := range entries { - if !entry.IsDir() { - continue - } - name := entry.Name() - // Skip non-ctrl_group entries. - if name == monGroupsDir || name == "info" || strings.HasPrefix(name, "mon_") { - continue - } - ctrlGroupMonDir := filepath.Join(r.resctrlPath, name, monGroupsDir) - r.cleanOrphanedInDir(ctrlGroupMonDir, state) - } -} - -// cleanOrphanedInDir removes mon_group directories in a specific mon_groups/ -// directory that look like pod UIDs but are not tracked in state. -func (r *resctrlOps) cleanOrphanedInDir(monGroupsPath string, state *podState) { - entries, err := os.ReadDir(monGroupsPath) - if err != nil { - if !errors.Is(err, os.ErrNotExist) { - log.Warnf("failed to read mon_groups directory %s: %v", monGroupsPath, err) - } - return - } - for _, entry := range entries { - if !entry.IsDir() { - continue - } - name := entry.Name() - // Only clean directories that look like pod UIDs. - u, err := uuid.Parse(name) - if err != nil { - continue - } - orphanDir := filepath.Join(monGroupsPath, name) - trackedDir := state.getMonGroupDir(u.String()) - if trackedDir == orphanDir { - // This is the active mon_group for this pod. - continue - } - log.Infof("removing orphaned mon_group %s", orphanDir) - if err := os.Remove(orphanDir); err != nil && !errors.Is(err, os.ErrNotExist) { - log.Warnf("failed to remove orphaned mon_group %s: %v", orphanDir, err) - } - } -} - -// isValidRDTClass returns true if the name is a safe resctrl ctrl_group name. -// It rejects path separators, dot-segments, and empty strings to prevent -// path traversal outside the resctrl mount. -func isValidRDTClass(name string) bool { - if name == "" || name == "." || name == ".." { - return false - } - for _, c := range name { - if c == '/' || c == 0 { - return false - } - } - return true -} diff --git a/cmd/plugins/resctrl-mon/resctrl_test.go b/cmd/plugins/resctrl-mon/resctrl_test.go deleted file mode 100644 index b74aa1c20..000000000 --- a/cmd/plugins/resctrl-mon/resctrl_test.go +++ /dev/null @@ -1,251 +0,0 @@ -// Copyright The NRI Plugins Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestCreateMonGroup_RootClass(t *testing.T) { - tmpDir := t.TempDir() - r := newResctrlOps(tmpDir) - - dir, err := r.createMonGroup("", "pod-uid-1") - require.NoError(t, err) - assert.Equal(t, filepath.Join(tmpDir, "mon_groups", "pod-uid-1"), dir) - - // Directory should exist. - info, err := os.Stat(dir) - require.NoError(t, err) - assert.True(t, info.IsDir()) -} - -func TestCreateMonGroup_WithRDTClass(t *testing.T) { - tmpDir := t.TempDir() - r := newResctrlOps(tmpDir) - require.NoError(t, os.Mkdir(filepath.Join(tmpDir, "BestEffort"), 0755)) - - dir, err := r.createMonGroup("BestEffort", "pod-uid-2") - require.NoError(t, err) - assert.Equal(t, filepath.Join(tmpDir, "BestEffort", "mon_groups", "pod-uid-2"), dir) - - info, err := os.Stat(dir) - require.NoError(t, err) - assert.True(t, info.IsDir()) -} - -func TestCreateMonGroup_MissingCtrlGroup(t *testing.T) { - tmpDir := t.TempDir() - r := newResctrlOps(tmpDir) - - // Attempt to create a mon_group under a non-existent ctrl_group. - _, err := r.createMonGroup("NoSuchClass", "pod-uid-3") - assert.Error(t, err) - assert.Contains(t, err.Error(), "ctrl_group") - - // Verify the ctrl_group was NOT created. - _, err = os.Stat(filepath.Join(tmpDir, "NoSuchClass")) - assert.True(t, os.IsNotExist(err)) -} - -func TestCreateMonGroup_Idempotent(t *testing.T) { - tmpDir := t.TempDir() - r := newResctrlOps(tmpDir) - - dir1, err := r.createMonGroup("", "pod-uid-1") - require.NoError(t, err) - - dir2, err := r.createMonGroup("", "pod-uid-1") - require.NoError(t, err) - - assert.Equal(t, dir1, dir2) -} - -func TestRemoveMonGroup(t *testing.T) { - tmpDir := t.TempDir() - r := newResctrlOps(tmpDir) - - dir, err := r.createMonGroup("", "pod-uid-1") - require.NoError(t, err) - - err = r.removeMonGroup(dir) - require.NoError(t, err) - - _, err = os.Stat(dir) - assert.True(t, os.IsNotExist(err)) -} - -func TestRemoveMonGroup_NotExist(t *testing.T) { - tmpDir := t.TempDir() - r := newResctrlOps(tmpDir) - - err := r.removeMonGroup(filepath.Join(tmpDir, "mon_groups", "nonexistent")) - assert.NoError(t, err) -} - -func TestWriteTaskPID(t *testing.T) { - tmpDir := t.TempDir() - r := newResctrlOps(tmpDir) - - dir, err := r.createMonGroup("", "pod-uid-1") - require.NoError(t, err) - - // In real resctrl, the kernel creates the tasks file when the - // mon_group directory is created. Simulate that here. - tasksFile := filepath.Join(dir, "tasks") - require.NoError(t, os.WriteFile(tasksFile, nil, 0644)) - - err = r.writeTaskPID(dir, 12345) - require.NoError(t, err) - - data, err := os.ReadFile(tasksFile) - require.NoError(t, err) - assert.Equal(t, "12345\n", string(data)) -} - -func TestCleanOrphanedMonGroups(t *testing.T) { - tmpDir := t.TempDir() - r := newResctrlOps(tmpDir) - state := newPodState() - - // Create a mon_group that IS tracked. - trackedUID := "a1b2c3d4-e5f6-7890-abcd-ef1234567890" - dir, err := r.createMonGroup("", trackedUID) - require.NoError(t, err) - state.addPod(trackedUID, dir) - - // Create a mon_group that is NOT tracked (orphan). - orphanUID := "deadbeef-dead-beef-dead-beefdeadbeef" - _, err = r.createMonGroup("", orphanUID) - require.NoError(t, err) - - r.cleanOrphanedMonGroups(state) - - // Tracked should still exist. - _, err = os.Stat(filepath.Join(tmpDir, "mon_groups", trackedUID)) - assert.NoError(t, err) - - // Orphan should be removed. - _, err = os.Stat(filepath.Join(tmpDir, "mon_groups", orphanUID)) - assert.True(t, os.IsNotExist(err)) -} - -func TestCleanOrphanedMonGroups_CtrlGroup(t *testing.T) { - tmpDir := t.TempDir() - r := newResctrlOps(tmpDir) - state := newPodState() - - // Create orphan under a ctrl_group. - orphanUID := "deadbeef-dead-beef-dead-beefdeadbeef" - require.NoError(t, os.Mkdir(filepath.Join(tmpDir, "BestEffort"), 0755)) - _, err := r.createMonGroup("BestEffort", orphanUID) - require.NoError(t, err) - - r.cleanOrphanedMonGroups(state) - - _, err = os.Stat(filepath.Join(tmpDir, "BestEffort", "mon_groups", orphanUID)) - assert.True(t, os.IsNotExist(err)) -} - -func TestCleanOrphanedMonGroups_StaleLocation(t *testing.T) { - tmpDir := t.TempDir() - r := newResctrlOps(tmpDir) - state := newPodState() - - podUID := "a1b2c3d4-e5f6-7890-abcd-ef1234567890" - - // Create a mon_group under BestEffort (simulates previous run). - require.NoError(t, os.Mkdir(filepath.Join(tmpDir, "BestEffort"), 0755)) - _, err := r.createMonGroup("BestEffort", podUID) - require.NoError(t, err) - - // Track the pod at the root class (simulates current run with different RDT class). - rootDir, err := r.createMonGroup("", podUID) - require.NoError(t, err) - state.addPod(podUID, rootDir) - - r.cleanOrphanedMonGroups(state) - - // Root mon_group (tracked) should still exist. - _, err = os.Stat(rootDir) - assert.NoError(t, err) - - // BestEffort mon_group (stale) should be removed. - _, err = os.Stat(filepath.Join(tmpDir, "BestEffort", "mon_groups", podUID)) - assert.True(t, os.IsNotExist(err)) -} - -func TestCleanOrphanedMonGroups_CompactUIDIsOrphaned(t *testing.T) { - tmpDir := t.TempDir() - r := newResctrlOps(tmpDir) - state := newPodState() - - // Simulate: pod is tracked under the canonical dashed UID (as ensureMonGroup stores it). - canonicalUID := "a1b2c3d4-e5f6-7890-abcd-ef1234567890" - canonicalDir, err := r.createMonGroup("", canonicalUID) - require.NoError(t, err) - state.addPod(canonicalUID, canonicalDir) - - // Simulate: a stale compact-form directory left by a previous run (e.g. CRI-O). - compactUID := "a1b2c3d4e5f678901234abcdef567890" - compactDir := filepath.Join(tmpDir, "mon_groups", compactUID) - require.NoError(t, os.Mkdir(compactDir, 0755)) - - r.cleanOrphanedMonGroups(state) - - // Canonical mon_group (tracked) must survive. - _, err = os.Stat(canonicalDir) - assert.NoError(t, err) - - // Compact-named directory is not tracked — must be removed as orphan. - _, err = os.Stat(compactDir) - assert.True(t, os.IsNotExist(err)) -} - -func TestIsValidRDTClass(t *testing.T) { - assert.True(t, isValidRDTClass("BestEffort")) - assert.True(t, isValidRDTClass("Guaranteed")) - assert.True(t, isValidRDTClass("COS1")) - assert.True(t, isValidRDTClass("my-class_v2")) - - assert.False(t, isValidRDTClass("")) - assert.False(t, isValidRDTClass(".")) - assert.False(t, isValidRDTClass("..")) - assert.False(t, isValidRDTClass("../../etc")) - assert.False(t, isValidRDTClass("foo/bar")) - assert.False(t, isValidRDTClass("class\x00name")) -} - -func TestCreateMonGroup_PathTraversal(t *testing.T) { - tmpDir := t.TempDir() - r := newResctrlOps(tmpDir) - - _, err := r.createMonGroup("../../etc", "a1b2c3d4-e5f6-7890-abcd-ef1234567890") - assert.Error(t, err) - assert.Contains(t, err.Error(), "invalid RDT class") - - _, err = r.createMonGroup("foo/bar", "a1b2c3d4-e5f6-7890-abcd-ef1234567890") - assert.Error(t, err) - assert.Contains(t, err.Error(), "invalid RDT class") - - _, err = r.createMonGroup("..", "a1b2c3d4-e5f6-7890-abcd-ef1234567890") - assert.Error(t, err) - assert.Contains(t, err.Error(), "invalid RDT class") -} diff --git a/cmd/plugins/resctrl-mon/state.go b/cmd/plugins/resctrl-mon/state.go deleted file mode 100644 index b3c58d8ab..000000000 --- a/cmd/plugins/resctrl-mon/state.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright The NRI Plugins Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import "sync" - -// podInfo tracks the mon_group directory and container set for a single pod. -type podInfo struct { - monGroupDir string - containers map[string]struct{} // container IDs -} - -// podState tracks all pods with active mon_groups. -type podState struct { - mu sync.Mutex - pods map[string]*podInfo // keyed by pod UID -} - -func newPodState() *podState { - return &podState{ - pods: make(map[string]*podInfo), - } -} - -// addPod registers a new pod with its mon_group directory. -// If the pod already exists, the existing entry is preserved. -func (s *podState) addPod(podUID, monGroupDir string) { - s.mu.Lock() - defer s.mu.Unlock() - if _, ok := s.pods[podUID]; ok { - return - } - s.pods[podUID] = &podInfo{ - monGroupDir: monGroupDir, - containers: make(map[string]struct{}), - } -} - -// addContainer adds a container ID to an existing pod's tracking. -func (s *podState) addContainer(podUID, containerID string) { - s.mu.Lock() - defer s.mu.Unlock() - if info, ok := s.pods[podUID]; ok { - info.containers[containerID] = struct{}{} - } -} - -// removeContainer removes a container ID from a pod's tracking. -func (s *podState) removeContainer(podUID, containerID string) { - s.mu.Lock() - defer s.mu.Unlock() - if info, ok := s.pods[podUID]; ok { - delete(info.containers, containerID) - } else { - log.Warnf("removeContainer: pod %s not tracked (container %s)", podUID, containerID) - } -} - -// removePod removes all tracking for a pod. -func (s *podState) removePod(podUID string) { - s.mu.Lock() - defer s.mu.Unlock() - delete(s.pods, podUID) -} - -// getMonGroupDir returns the mon_group directory for a pod, or empty string. -func (s *podState) getMonGroupDir(podUID string) string { - s.mu.Lock() - defer s.mu.Unlock() - if info, ok := s.pods[podUID]; ok { - return info.monGroupDir - } - return "" -} - -// podHasNoContainers returns true if the pod has no remaining containers. -// Returns true for untracked pods since there is nothing to protect. -func (s *podState) podHasNoContainers(podUID string) bool { - s.mu.Lock() - defer s.mu.Unlock() - if info, ok := s.pods[podUID]; ok { - return len(info.containers) == 0 - } - log.Warnf("podHasNoContainers: pod %s not tracked, treating as empty", podUID) - return true -} - -// hasPod returns true if the pod UID is being tracked. -func (s *podState) hasPod(podUID string) bool { - s.mu.Lock() - defer s.mu.Unlock() - _, ok := s.pods[podUID] - return ok -} - -// podCount returns the number of tracked pods. -func (s *podState) podCount() int { - s.mu.Lock() - defer s.mu.Unlock() - return len(s.pods) -} diff --git a/cmd/plugins/resctrl-mon/telemetry.go b/cmd/plugins/resctrl-mon/telemetry.go new file mode 100644 index 000000000..793567e3b --- /dev/null +++ b/cmd/plugins/resctrl-mon/telemetry.go @@ -0,0 +1,243 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "net" + "net/http" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" + promexp "go.opentelemetry.io/otel/exporters/prometheus" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" +) + +// telemetryConfig holds the OTel exporter configuration for the plugin. +type telemetryConfig struct { + Prometheus struct { + Enabled bool `json:"enabled"` + ListenAddress string `json:"listenAddress"` + Namespace string `json:"namespace"` + } `json:"prometheus"` + OTLP struct { + Enabled bool `json:"enabled"` + Endpoint string `json:"endpoint"` + Protocol string `json:"protocol"` + Interval string `json:"interval"` + Insecure bool `json:"insecure"` + } `json:"otlp"` + PerfCounters struct { + Enabled bool `json:"enabled"` + Include []string `json:"include"` + Exclude []string `json:"exclude"` + } `json:"perfCounters"` + ResourceAttributes map[string]string `json:"resourceAttributes"` +} + +// telemetryState holds the running telemetry components for graceful shutdown. +type telemetryState struct { + provider *metric.MeterProvider + server *http.Server + promListener net.Listener +} + +// defaultTelemetryConfig returns the config defaults (Prometheus on :9100). +func defaultTelemetryConfig() telemetryConfig { + var cfg telemetryConfig + cfg.Prometheus.Enabled = true + cfg.Prometheus.ListenAddress = ":9100" + return cfg +} + +// validateTelemetryConfig checks the telemetry config for invalid combinations. +func validateTelemetryConfig(cfg *telemetryConfig) error { + if cfg.OTLP.Enabled && cfg.OTLP.Endpoint == "" { + return fmt.Errorf("telemetry: otlp.enabled requires a non-empty endpoint") + } + if cfg.OTLP.Protocol == "" { + cfg.OTLP.Protocol = "grpc" + } + if cfg.OTLP.Protocol != "grpc" && cfg.OTLP.Protocol != "http" { + return fmt.Errorf("telemetry: otlp.protocol must be \"grpc\" or \"http\", got %q", cfg.OTLP.Protocol) + } + if cfg.OTLP.Interval == "" { + cfg.OTLP.Interval = "15s" + } + if _, err := time.ParseDuration(cfg.OTLP.Interval); err != nil { + return fmt.Errorf("telemetry: otlp.interval %q: %w", cfg.OTLP.Interval, err) + } + if len(cfg.PerfCounters.Include) > 0 && len(cfg.PerfCounters.Exclude) > 0 { + return fmt.Errorf("telemetry: perfCounters.include and perfCounters.exclude are mutually exclusive") + } + if cfg.Prometheus.ListenAddress == "" { + cfg.Prometheus.ListenAddress = ":9100" + } + return nil +} + +// newTelemetry creates the MeterProvider with configured exporters. +func newTelemetry(ctx context.Context, cfg telemetryConfig) (*telemetryState, error) { + var opts []metric.Option + + res, err := resource.New(ctx, + resource.WithAttributes(resourceAttrs(cfg.ResourceAttributes)...), + ) + if err != nil { + return nil, fmt.Errorf("telemetry: failed to create resource: %w", err) + } + opts = append(opts, metric.WithResource(res)) + + state := &telemetryState{} + + if cfg.Prometheus.Enabled { + reg := prometheus.NewRegistry() + peOpts := []promexp.Option{promexp.WithRegisterer(reg)} + if cfg.Prometheus.Namespace != "" { + peOpts = append(peOpts, promexp.WithNamespace(cfg.Prometheus.Namespace)) + } + pe, err := promexp.New(peOpts...) + if err != nil { + return nil, fmt.Errorf("telemetry: prometheus exporter: %w", err) + } + opts = append(opts, metric.WithReader(pe)) + + // Bind synchronously so a failure (e.g. address already in use) is + // returned to the caller instead of only surfacing asynchronously from + // the serving goroutine, which would leave the endpoint advertised for + // scraping with nothing actually listening. + ln, err := net.Listen("tcp", cfg.Prometheus.ListenAddress) + if err != nil { + return nil, fmt.Errorf("telemetry: prometheus listen on %s: %w", cfg.Prometheus.ListenAddress, err) + } + state.promListener = ln + + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{})) + state.server = &http.Server{Addr: cfg.Prometheus.ListenAddress, Handler: mux} + } + + if cfg.OTLP.Enabled { + interval, _ := time.ParseDuration(cfg.OTLP.Interval) + exp, err := newOTLPExporter(ctx, cfg) + if err != nil { + if state.promListener != nil { + _ = state.promListener.Close() + } + return nil, fmt.Errorf("telemetry: OTLP exporter: %w", err) + } + opts = append(opts, metric.WithReader( + metric.NewPeriodicReader(exp, metric.WithInterval(interval)), + )) + log.Infof("telemetry: OTLP push enabled → %s (%s, interval=%s)", + cfg.OTLP.Endpoint, cfg.OTLP.Protocol, cfg.OTLP.Interval) + } + + state.provider = metric.NewMeterProvider(opts...) + + // All fallible setup has succeeded; only now start serving the (already + // bound) Prometheus listener. + if state.promListener != nil { + go func() { + if err := state.server.Serve(state.promListener); err != nil && err != http.ErrServerClosed { + log.Warnf("telemetry: prometheus server error: %v", err) + } + }() + log.Infof("telemetry: Prometheus endpoint listening on %s/metrics", state.promListener.Addr()) + } + + return state, nil +} + +// shutdown gracefully shuts down the telemetry stack. +func (t *telemetryState) shutdown(ctx context.Context) { + if t.provider != nil { + if err := t.provider.Shutdown(ctx); err != nil { + log.Warnf("telemetry: provider shutdown: %v", err) + } + } + if t.server != nil { + if err := t.server.Shutdown(ctx); err != nil { + log.Warnf("telemetry: http server shutdown: %v", err) + } + } +} + +// newOTLPExporter creates a gRPC or HTTP OTLP metric exporter. +func newOTLPExporter(ctx context.Context, cfg telemetryConfig) (metric.Exporter, error) { + switch cfg.OTLP.Protocol { + case "http": + opts := []otlpmetrichttp.Option{ + otlpmetrichttp.WithEndpoint(cfg.OTLP.Endpoint), + } + if cfg.OTLP.Insecure { + opts = append(opts, otlpmetrichttp.WithInsecure()) + } + return otlpmetrichttp.New(ctx, opts...) + default: // grpc + opts := []otlpmetricgrpc.Option{ + otlpmetricgrpc.WithEndpoint(cfg.OTLP.Endpoint), + } + if cfg.OTLP.Insecure { + opts = append(opts, otlpmetricgrpc.WithInsecure()) + } + return otlpmetricgrpc.New(ctx, opts...) + } +} + +// resourceAttrs builds OTel resource attributes from the config map. +func resourceAttrs(m map[string]string) []attribute.KeyValue { + attrs := []attribute.KeyValue{ + semconv.ServiceName("nri-resctrl-mon"), + } + for k, v := range m { + if k == "service.name" { + // Override the default service name. + attrs[0] = semconv.ServiceName(v) + continue + } + attrs = append(attrs, attribute.String(k, v)) + } + return attrs +} + +// startTelemetry initializes the MeterProvider and registers metrics instruments. +func (p *plugin) startTelemetry(ctx context.Context) error { + cfg := p.config.Telemetry + if err := validateTelemetryConfig(&cfg); err != nil { + return err + } + state, err := newTelemetry(ctx, cfg) + if err != nil { + return err + } + p.telemetry = state + + meter := state.provider.Meter("nri-resctrl-mon") + reg, err := setupMetrics(p.mgr, cfg, meter) + if err != nil { + state.shutdown(ctx) + return fmt.Errorf("metrics registration: %w", err) + } + p.metrics = reg + return nil +} diff --git a/cmd/plugins/resctrl-mon/telemetry_test.go b/cmd/plugins/resctrl-mon/telemetry_test.go new file mode 100644 index 000000000..2bd308426 --- /dev/null +++ b/cmd/plugins/resctrl-mon/telemetry_test.go @@ -0,0 +1,336 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "io" + "net/http" + "os" + "path/filepath" + "testing" + "time" + + "github.com/intel/goresctrl/pkg/monitor" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// setupTestResctrl creates a minimal resctrl-like filesystem for testing. +// Returns the root path and a cleanup function. +func setupTestResctrl(t *testing.T, groups map[string]map[string]map[string]string) string { + t.Helper() + root := t.TempDir() + monGroupsDir := filepath.Join(root, "mon_groups") + require.NoError(t, os.MkdirAll(monGroupsDir, 0o755)) + + // Collect all domains/counters to also create root-level mon_data + // (used by RegisterOTelInstruments to discover available instruments). + allDomains := make(map[string]map[string]string) + + for groupName, domains := range groups { + groupDir := filepath.Join(monGroupsDir, groupName) + require.NoError(t, os.MkdirAll(groupDir, 0o755)) + // Create tasks file. + require.NoError(t, os.WriteFile(filepath.Join(groupDir, "tasks"), []byte(""), 0o644)) + monDataDir := filepath.Join(groupDir, "mon_data") + for domain, counters := range domains { + domDir := filepath.Join(monDataDir, domain) + require.NoError(t, os.MkdirAll(domDir, 0o755)) + for file, value := range counters { + require.NoError(t, os.WriteFile(filepath.Join(domDir, file), []byte(value+"\n"), 0o644)) + } + // Merge into allDomains. + if allDomains[domain] == nil { + allDomains[domain] = make(map[string]string) + } + for file, value := range counters { + allDomains[domain][file] = value + } + } + } + + // Create root-level mon_data for instrument discovery. + rootMonData := filepath.Join(root, "mon_data") + for domain, counters := range allDomains { + domDir := filepath.Join(rootMonData, domain) + require.NoError(t, os.MkdirAll(domDir, 0o755)) + for file, value := range counters { + require.NoError(t, os.WriteFile(filepath.Join(domDir, file), []byte(value+"\n"), 0o644)) + } + } + + return root +} + +func TestTelemetryPrometheusEndpoint(t *testing.T) { + podUID := "12345678-1234-1234-1234-123456789abc" + root := setupTestResctrl(t, map[string]map[string]map[string]string{ + podUID: { + "mon_L3_00": { + "llc_occupancy": "4096", + "mbm_local_bytes": "1000000", + "mbm_total_bytes": "2000000", + }, + "mon_PERF_PKG_00": { + "core_energy": "54446119.644974", + "activity": "12345.6789", + "c6_res": "9999", + }, + }, + }) + + mgr, err := monitor.New(monitor.Options{ + ResctrlRoot: root, + KeyValidator: monitor.PodUIDValidator, + KeyCanonicalizer: monitor.CanonicalizePodUID, + }) + require.NoError(t, err) + _, err = mgr.EnsureGroup(podUID, "") + require.NoError(t, err) + + // Use a random port to avoid conflicts. + cfg := defaultTelemetryConfig() + cfg.Prometheus.ListenAddress = "127.0.0.1:0" + cfg.PerfCounters.Enabled = false // default: suppress perf counters + + state, err := newTelemetry(context.Background(), cfg) + require.NoError(t, err) + defer state.shutdown(context.Background()) + + // Get the actual port from the listener. + // Since we used :0, we need to start with a listener first. + // Workaround: use a fixed high port for testing. + state.shutdown(context.Background()) + + cfg.Prometheus.ListenAddress = "127.0.0.1:19100" + state, err = newTelemetry(context.Background(), cfg) + require.NoError(t, err) + defer state.shutdown(context.Background()) + + meter := state.provider.Meter("nri-resctrl-mon-test") + _, err = setupMetrics(mgr, cfg, meter) + require.NoError(t, err) + + // Wait for server to be ready. + time.Sleep(50 * time.Millisecond) + + resp, err := http.Get("http://127.0.0.1:19100/metrics") + require.NoError(t, err) + defer func() { _ = resp.Body.Close() }() + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + metrics := string(body) + + // Verify core instruments are registered and produce correct Prometheus names. + // Instrument names are derived from domain + counter: + // mon_L3_00/llc_occupancy → l3.llc.occupancy (unit By) → l3_llc_occupancy_bytes + // mon_L3_00/mbm_local_bytes → l3.mbm.local.bytes (unit By) → l3_mbm_local_bytes_total + // mon_L3_00/mbm_total_bytes → l3.mbm.total.bytes (unit By) → l3_mbm_bytes_total + // (the otlptranslator collapses the semantic "total" into the counter + // "_total" suffix; the local counter above disambiguates it) + // mon_PERF_PKG_00/core_energy → perf.core.energy (unit J) → perf_core_energy_joules_total + // mon_PERF_PKG_00/activity → perf.activity (unit farads) → perf_activity_farads_total + assert.Contains(t, metrics, "l3_llc_occupancy_bytes") + assert.Contains(t, metrics, "l3_mbm_local_bytes_total") + assert.Contains(t, metrics, "l3_mbm_bytes_total") + assert.Contains(t, metrics, "perf_core_energy_joules_total") + assert.Contains(t, metrics, "perf_activity_farads_total") + + // Verify float64 fidelity through the full pipeline. + assert.Contains(t, metrics, "5.4446119644974e+07", + "core_energy float64 value must survive OTel→Prometheus rendering") +} + +func TestFloat64Fidelity(t *testing.T) { + podUID := "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + root := setupTestResctrl(t, map[string]map[string]map[string]string{ + podUID: { + "mon_PERF_PKG_00": { + "core_energy": "54446119.644974", + }, + }, + }) + + mgr, err := monitor.New(monitor.Options{ + ResctrlRoot: root, + KeyValidator: monitor.PodUIDValidator, + KeyCanonicalizer: monitor.CanonicalizePodUID, + }) + require.NoError(t, err) + _, err = mgr.EnsureGroup(podUID, "") + require.NoError(t, err) + + readings, err := mgr.ReadCounters(podUID) + require.NoError(t, err) + require.Len(t, readings, 1) + assert.Equal(t, 54446119.644974, readings[0].Value, + "float64 value must preserve kernel precision without integer truncation") +} + +func TestPerfCountersGate(t *testing.T) { + podUID := "11111111-2222-3333-4444-555555555555" + root := setupTestResctrl(t, map[string]map[string]map[string]string{ + podUID: { + "mon_PERF_PKG_00": { + "core_energy": "100.5", + "activity": "50.0", + "c6_res": "9999", + "unhalted_core_cycles": "123456", + }, + "mon_L3_00": { + "llc_occupancy": "8192", + }, + }, + }) + + mgr, err := monitor.New(monitor.Options{ + ResctrlRoot: root, + KeyValidator: monitor.PodUIDValidator, + KeyCanonicalizer: monitor.CanonicalizePodUID, + }) + require.NoError(t, err) + _, err = mgr.EnsureGroup(podUID, "") + require.NoError(t, err) + + readings, err := mgr.ReadCounters(podUID) + require.NoError(t, err) + + t.Run("disabled suppresses perf-only counters", func(t *testing.T) { + cfg := defaultTelemetryConfig() + cfg.PerfCounters.Enabled = false + filter := perfCounterFilter(cfg) + + var allowed []string + for _, r := range readings { + if filter(r) { + allowed = append(allowed, r.Name) + } + } + // core_energy, activity (from PERF_PKG) + llc_occupancy (from L3) should pass. + assert.Contains(t, allowed, "core_energy") + assert.Contains(t, allowed, "activity") + assert.Contains(t, allowed, "llc_occupancy") + // c6_res and unhalted_core_cycles should be blocked. + assert.NotContains(t, allowed, "c6_res") + assert.NotContains(t, allowed, "unhalted_core_cycles") + }) + + t.Run("enabled allows all perf counters", func(t *testing.T) { + cfg := defaultTelemetryConfig() + cfg.PerfCounters.Enabled = true + filter := perfCounterFilter(cfg) + + var allowed []string + for _, r := range readings { + if filter(r) { + allowed = append(allowed, r.Name) + } + } + assert.Contains(t, allowed, "c6_res") + assert.Contains(t, allowed, "unhalted_core_cycles") + }) + + t.Run("include list filters", func(t *testing.T) { + cfg := defaultTelemetryConfig() + cfg.PerfCounters.Enabled = true + cfg.PerfCounters.Include = []string{"unhalted_*"} + filter := perfCounterFilter(cfg) + + var allowed []string + for _, r := range readings { + if filter(r) { + allowed = append(allowed, r.Name) + } + } + assert.Contains(t, allowed, "unhalted_core_cycles") + assert.Contains(t, allowed, "core_energy") // always allowed (core AET) + assert.NotContains(t, allowed, "c6_res") // not in include list + }) +} + +func TestControlGroupOf(t *testing.T) { + tests := []struct { + path string + want string + }{ + {"/sys/fs/resctrl/mon_groups/abc-123", ""}, + {"/sys/fs/resctrl/COS1/mon_groups/abc-123", "COS1"}, + {"/sys/fs/resctrl/my-class/mon_groups/abc-123", "my-class"}, + } + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + assert.Equal(t, tt.want, controlGroupOf(tt.path)) + }) + } +} + +func TestSourceFor(t *testing.T) { + assert.Equal(t, "pod", sourceFor("12345678-1234-1234-1234-123456789abc")) + assert.Equal(t, "other", sourceFor("machine-qemu-1-vm")) + assert.Equal(t, "other", sourceFor("not-a-uuid")) +} + +func TestValidateTelemetryConfig(t *testing.T) { + t.Run("valid defaults", func(t *testing.T) { + cfg := defaultTelemetryConfig() + assert.NoError(t, validateTelemetryConfig(&cfg)) + }) + + t.Run("otlp enabled requires endpoint", func(t *testing.T) { + cfg := defaultTelemetryConfig() + cfg.OTLP.Enabled = true + assert.Error(t, validateTelemetryConfig(&cfg)) + }) + + t.Run("invalid protocol", func(t *testing.T) { + cfg := defaultTelemetryConfig() + cfg.OTLP.Enabled = true + cfg.OTLP.Endpoint = "localhost:4317" + cfg.OTLP.Protocol = "websocket" + assert.Error(t, validateTelemetryConfig(&cfg)) + }) + + t.Run("include and exclude mutually exclusive", func(t *testing.T) { + cfg := defaultTelemetryConfig() + cfg.PerfCounters.Include = []string{"c6_res"} + cfg.PerfCounters.Exclude = []string{"c1_res"} + assert.Error(t, validateTelemetryConfig(&cfg)) + }) +} + +func TestInstrumentNaming(t *testing.T) { + // Verify the library's derived naming matches expectations. + t.Run("L3 counters match rdt convention", func(t *testing.T) { + assert.Equal(t, "l3.llc.occupancy", monitor.InstrumentName("mon_L3_00", "llc_occupancy")) + assert.Equal(t, "l3.mbm.local.bytes", monitor.InstrumentName("mon_L3_00", "mbm_local_bytes")) + assert.Equal(t, "l3.mbm.total.bytes", monitor.InstrumentName("mon_L3_00", "mbm_total_bytes")) + }) + + t.Run("PERF_PKG counters", func(t *testing.T) { + assert.Equal(t, "perf.core.energy", monitor.InstrumentName("mon_PERF_PKG_00", "core_energy")) + assert.Equal(t, "perf.activity", monitor.InstrumentName("mon_PERF_PKG_00", "activity")) + assert.Equal(t, "perf.c1.res", monitor.InstrumentName("mon_PERF_PKG_00", "c1_res")) + }) +} + +func TestMatchGlob(t *testing.T) { + assert.True(t, matchGlob("unhalted_*", "unhalted_core_cycles")) + assert.True(t, matchGlob("unhalted_*", "unhalted_ref_cycles")) + assert.False(t, matchGlob("unhalted_*", "c6_res")) + assert.True(t, matchGlob("c6_res", "c6_res")) + assert.True(t, matchGlob("*_bytes", "mbm_local_bytes")) +} diff --git a/deployment/helm/resctrl-mon/README.md b/deployment/helm/resctrl-mon/README.md index b77809acf..212e90042 100644 --- a/deployment/helm/resctrl-mon/README.md +++ b/deployment/helm/resctrl-mon/README.md @@ -121,3 +121,66 @@ customize with their own values, along with the default values. | `affinity` | [] | specify node affinity | | `nodeSelector` | [] | specify node selector labels | | `podPriorityClassNodeCritical` | true | enable [marking Pod as node critical](https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/#marking-pod-as-critical) | + +### Telemetry options + +| Name | Default | Description | +| -------------------------------------- | --------- | ------------------------------------------------------------------ | +| `telemetry.prometheus.enabled` | `true` | expose a `/metrics` Prometheus endpoint | +| `telemetry.prometheus.listenAddress` | `":9100"` | address:port for the Prometheus HTTP listener | +| `telemetry.prometheus.scrapeInterval` | `"15s"` | recommended scrape interval (set via pod annotation hint) | +| `telemetry.prometheus.namespace` | `""` | Prometheus metric prefix (empty = `resctrl_`) | +| `telemetry.otlp.enabled` | `false` | push metrics via OTLP | +| `telemetry.otlp.endpoint` | `""` | OTLP receiver endpoint (e.g. `otel-collector:4317`) | +| `telemetry.otlp.protocol` | `grpc` | `grpc` or `http` | +| `telemetry.otlp.interval` | `15s` | OTLP export interval | +| `telemetry.otlp.insecure` | `true` | disable TLS for OTLP connection | +| `telemetry.perfCounters.enabled` | `false` | gate `rdt=perf` counters (c1_res, stalls_*, etc.) | +| `telemetry.perfCounters.include` | `[]` | glob patterns for counters to include | +| `telemetry.perfCounters.exclude` | `[]` | glob patterns for counters to exclude | +| `telemetry.resourceAttributes` | `{}` | static OTel resource attributes added to all metrics | + +## Prometheus Integration + +The DaemonSet pods are annotated with `prometheus.io/scrape: "true"` so that +standard Prometheus service-discovery configurations will pick them up +automatically. The `prometheus.io/interval` annotation is set to the +configured `telemetry.prometheus.scrapeInterval` (default 15s) as a hint, +but note that most Prometheus deployments do not honor this annotation +without additional relabel configuration. + +If you need a specific scrape interval, configure a dedicated scrape job +in your Prometheus configuration with the desired `scrape_interval`. + +## Runtime Requirements + +| Component | Minimum Version | Notes | +| ---------------- | --------------- | --------------------------------------------------------------------- | +| Linux kernel | 7.0+ | Required for AET (`rdt=perf` Kconfig). CMT/MBM works on 5.x+. | +| containerd | 1.7.0+ | NRI support required. | +| CRI-O | 1.36.0+ | Provides container PIDs via NRI `LinuxContainer.Pid`. | +| Kubernetes | 1.24+ | DaemonSet and NRI socket conventions. | +| CPU | Intel RDT | CMT/MBM for bandwidth/LLC counters; AET for energy/perf counters. | + +### Kernel feature matrix + +| Counter family | Kernel Kconfig | Available since | +| ------------------------------- | ----------------------------- | --------------- | +| `llc_occupancy`, `mbm_*` | `CONFIG_X86_CPU_RESCTRL` | 5.x | +| `c1_res`, `stalls_*`, `energy_*` | `CONFIG_X86_CPU_RESCTRL` + `rdt=perf` boot param | 7.0 (under review) | + +## Optional: OTel Collector sidecar + +When using OTLP push mode (`telemetry.otlp.enabled=true`), you may deploy an +OTel Collector agent to receive, enrich, and fan out the metrics. Reference +manifests are provided in `optional/`: + +```sh +kubectl apply -f optional/otel-collector-rbac.yaml +kubectl apply -f optional/otel-collector-agent.yaml +``` + +The reference config uses the `k8sattributes` processor to attach pod/namespace +labels and a Prometheus exporter on port 8889. Customize +`otel-collector-agent.yaml` to add additional exporters (e.g. `otlphttp` to a +remote backend). diff --git a/deployment/helm/resctrl-mon/optional/grafana-resctrl-perf-counters.json b/deployment/helm/resctrl-mon/optional/grafana-resctrl-perf-counters.json new file mode 100644 index 000000000..d07f05714 --- /dev/null +++ b/deployment/helm/resctrl-mon/optional/grafana-resctrl-perf-counters.json @@ -0,0 +1,996 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(kube_pod_info, pod)", + "hide": 0, + "includeAll": true, + "label": "Pod", + "multi": true, + "name": "pod_name", + "query": { + "qryType": 1, + "query": "label_values(kube_pod_info, pod)" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(kube_pod_info, namespace)", + "hide": 0, + "includeAll": true, + "label": "Namespace", + "multi": true, + "name": "namespace", + "query": { + "qryType": 1, + "query": "label_values(kube_pod_info, namespace)" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + } + ] + }, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total power across all monitored pods", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#1a5cff", + "value": null + }, + { + "color": "#22d66a", + "value": 0 + }, + { + "color": "#f0a030", + "value": 100 + }, + { + "color": "#ff5c5c", + "value": 200 + } + ] + }, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate({__name__=\"perf.core.energy_joules_total\", \"resctrl.group.source\"=\"pod\"}[$__rate_interval]))", + "interval": "5s", + "legendFormat": "{{k8s.pod.uid}}", + "refId": "A" + } + ], + "title": "Total Pod Power", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total CPU activity rate across all monitored pods", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#1a5cff", + "value": null + }, + { + "color": "#22d66a", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 5 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate({__name__=\"perf.activity_farads_total\", \"resctrl.group.source\"=\"pod\"}[$__rate_interval]))", + "interval": "5s", + "legendFormat": "{{k8s.pod.uid}}", + "refId": "A" + } + ], + "title": "Total Activity Rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total unhalted core cycles/sec across all monitored pods", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#1a5cff", + "value": null + }, + { + "color": "#22d66a", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 9 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate({__name__=\"perf.unhalted.core.cycles_total\", \"resctrl.group.source\"=\"pod\"}[$__rate_interval]))", + "interval": "5s", + "legendFormat": "{{k8s.pod.uid}}", + "refId": "A" + } + ], + "title": "Total Core Cycles/s", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total retired micro-ops/sec across all monitored pods", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#1a5cff", + "value": null + }, + { + "color": "#22d66a", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 13 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate({__name__=\"perf.uops.retired_total\", \"resctrl.group.source\"=\"pod\"}[$__rate_interval]))", + "interval": "5s", + "legendFormat": "{{k8s.pod.uid}}", + "refId": "A" + } + ], + "title": "Total \u00b5ops Retired/s", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 10, + "title": "Power (Energy Rate)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Per-pod power consumption derived from Intel AET energy counter", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "lineWidth": 1, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (pod) (\n rate({__name__=\"perf.core.energy_joules_total\"}[$__rate_interval])\n * on(\"k8s.pod.uid\") group_left(pod, namespace)\nlabel_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\")\n)", + "interval": "5s", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Pod Power (Watts)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 20, + "title": "Activity Rate", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Per-pod CPU activity rate (capacitance proxy from AET)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "lineWidth": 1, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 21, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (pod) (\n rate({__name__=\"perf.activity_farads_total\"}[$__rate_interval])\n * on(\"k8s.pod.uid\") group_left(pod, namespace)\nlabel_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\")\n)", + "interval": "5s", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Pod Activity Rate (Farads/s)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 30, + "title": "Frequency Scaling Factor", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Ratio of unhalted core cycles to reference cycles per pod. 1.0 = base frequency, >1.0 = turbo, <1.0 = throttled.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "scaling factor (1.0 = base freq)", + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "lineWidth": 1, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "green", + "value": 1.0 + } + ] + }, + "unit": "none", + "min": 0, + "decimals": 3 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 31, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (pod) (\n (\n sum by (\"k8s.pod.uid\") (rate({__name__=\"perf.unhalted.core.cycles_total\"}[$__rate_interval]))\n / clamp_min(sum by (\"k8s.pod.uid\") (rate({__name__=\"perf.unhalted.ref.cycles_total\"}[$__rate_interval])), 1)\n )\n * on(\"k8s.pod.uid\") group_left(pod, namespace)\n label_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\")\n)", + "interval": "5s", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Frequency Scaling Factor (core/ref cycles ratio)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 44 + }, + "id": 40, + "title": "\u00b5ops Retired", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Per-pod retired micro-operations per second", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "lineWidth": 1, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 45 + }, + "id": 41, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (pod) (\n rate({__name__=\"perf.uops.retired_total\"}[$__rate_interval])\n * on(\"k8s.pod.uid\") group_left(pod, namespace)\nlabel_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\")\n)", + "interval": "5s", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "\u00b5ops Retired/s", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 53 + }, + "id": 50, + "title": "C-state Residency", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Per-pod C1 residency rate", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "avg cores in C1", + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "lineWidth": 1, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 54 + }, + "id": 51, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (pod) (\n rate({__name__=\"perf.c1.res_total\"}[$__rate_interval])\n * on(\"k8s.pod.uid\") group_left(pod, namespace)\nlabel_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\")\n)", + "interval": "5s", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "C1 Residency Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Per-pod C6 residency rate", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "avg cores in C6", + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "lineWidth": 1, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 62 + }, + "id": 52, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (pod) (\n rate({__name__=\"perf.c6.res_total\"}[$__rate_interval])\n * on(\"k8s.pod.uid\") group_left(pod, namespace)\nlabel_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\")\n)", + "interval": "5s", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "C6 Residency Rate", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 70 + }, + "id": 60, + "title": "LLC Stalls", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Per-pod LLC hit stalls per second", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "lineWidth": 1, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 71 + }, + "id": 61, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (pod) (\n rate({__name__=\"perf.stalls.llc.hit_total\"}[$__rate_interval])\n * on(\"k8s.pod.uid\") group_left(pod, namespace)\nlabel_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\")\n)", + "interval": "5s", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "LLC Hit Stalls/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Per-pod LLC miss stalls per second", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "lineWidth": 1, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 79 + }, + "id": 62, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (pod) (\n rate({__name__=\"perf.stalls.llc.miss_total\"}[$__rate_interval])\n * on(\"k8s.pod.uid\") group_left(pod, namespace)\nlabel_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\")\n)", + "interval": "5s", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "LLC Miss Stalls/s", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": [ + "aet", + "energy", + "kubernetes", + "resctrl", + "perf" + ], + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Kubernetes Pod Perf Counters (Intel AET / resctrl-mon)", + "uid": "resctrl-perf-counters", + "version": 3 +} diff --git a/deployment/helm/resctrl-mon/optional/grafana-resctrl-pod-energy.json b/deployment/helm/resctrl-mon/optional/grafana-resctrl-pod-energy.json new file mode 100644 index 000000000..15d2bff0c --- /dev/null +++ b/deployment/helm/resctrl-mon/optional/grafana-resctrl-pod-energy.json @@ -0,0 +1,1088 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Sum of power consumed by all monitored pods (from Intel AET via resctrl)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#1a5cff", + "value": null + }, + { + "color": "#22d66a", + "value": 0 + }, + { + "color": "#f0a030", + "value": 100 + }, + { + "color": "#ff5c5c", + "value": 200 + } + ] + }, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate({__name__=\"perf.core.energy_joules_total\", \"resctrl.group.source\"=\"pod\"}[$__rate_interval]))", + "legendFormat": "Total Pod Power", + "refId": "A", + "interval": "5s" + } + ], + "title": "Total Pod Power", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Energy consumed by all monitored pods in the displayed time window (Wh)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#22d66a", + "value": null + } + ] + }, + "unit": "watth" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(increase({__name__=\"perf.core.energy_joules_total\", \"resctrl.group.source\"=\"pod\"}[$__range])) / 3600", + "legendFormat": "Total Energy", + "refId": "A", + "interval": "5s" + } + ], + "title": "Total Pod Energy", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Percentage of total activity attributable to monitored pods (AET activity counters, measured in Farads)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#22d66a", + "value": null + }, + { + "color": "#f0a030", + "value": 50 + }, + { + "color": "#ff5c5c", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate({__name__=\"perf.activity_farads_total\", \"resctrl.group.source\"=\"pod\"}[$__rate_interval])) / sum(rate({__name__=\"perf.activity_farads_total\"}[$__rate_interval])) * 100", + "legendFormat": "Pod Activity Share", + "refId": "A", + "interval": "5s" + } + ], + "title": "Pod Activity Share", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 101, + "title": "Per-Pod Power", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Power consumption per pod from Intel AET core_energy counters", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Power (W)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 16, + "x": 0, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod) (rate({__name__=\"perf.core.energy_joules_total\", \"resctrl.group.source\"=\"pod\"}[$__rate_interval]) * on(\"k8s.pod.uid\") group_left(pod, namespace) label_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info[$__range])), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\"))", + "legendFormat": "{{pod_name}}", + "refId": "A", + "interval": "5s" + } + ], + "title": "Per-Pod Power Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total energy (Joules) consumed by each workload over the selected range (donut chart). Pods sharing a common name are summed; pods that started and stopped within the range are included.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "joule" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 8 + }, + "id": 5, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "bottom", + "values": [ + "value", + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (workload) (increase({__name__=\"perf.core.energy_joules_total\", \"resctrl.group.source\"=\"pod\"}[$__range]) * on(\"k8s.pod.uid\") group_left(workload) label_replace(label_replace(label_replace(label_replace(max by (uid, pod, namespace, created_by_name) (last_over_time(kube_pod_info[$__range])), \"workload\", \"$1\", \"pod\", \"(.+)\"), \"workload\", \"$1\", \"created_by_name\", \"(.+)\"), \"workload\", \"$1\", \"workload\", \"(.+?)-[bcdfghjklmnpqrstvwxz2-9]{6,10}$\"), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\"))", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "title": "Energy Breakdown by Pod", + "type": "piechart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 102, + "title": "Per-Pod Activity", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Activity rate (F/s) per pod from AET activity counters. Activity is measured in Farads and represents frequency-independent work done.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Activity (F/s)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": " F/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 16, + "x": 0, + "y": 19 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod) (rate({__name__=\"perf.activity_farads_total\", \"resctrl.group.source\"=\"pod\"}[$__rate_interval]) * on(\"k8s.pod.uid\") group_left(pod, namespace) label_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info[$__range])), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\"))", + "legendFormat": "{{pod_name}}", + "refId": "A", + "interval": "5s" + } + ], + "title": "Per-Pod Activity Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total activity (Farads) accumulated by each workload over the selected range (donut chart). Activity is frequency-independent work. Pods sharing a common name are summed; pods that started and stopped within the range are included.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": " F" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 7, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "bottom", + "values": [ + "value", + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (workload) (increase({__name__=\"perf.activity_farads_total\", \"resctrl.group.source\"=\"pod\"}[$__range]) * on(\"k8s.pod.uid\") group_left(workload) label_replace(label_replace(label_replace(label_replace(max by (uid, pod, namespace, created_by_name) (last_over_time(kube_pod_info[$__range])), \"workload\", \"$1\", \"pod\", \"(.+)\"), \"workload\", \"$1\", \"created_by_name\", \"(.+)\"), \"workload\", \"$1\", \"workload\", \"(.+?)-[bcdfghjklmnpqrstvwxz2-9]{6,10}$\"), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\"))", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "title": "Activity Breakdown by Pod", + "type": "piechart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 103, + "title": "Pod Details", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "All monitored pods with their current power draw, activity rate, and cumulative energy", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "fieldMinMax": true + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Pod" + }, + "properties": [ + { + "id": "custom.width", + "value": 280 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Namespace" + }, + "properties": [ + { + "id": "custom.width", + "value": 120 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Power (W)" + }, + "properties": [ + { + "id": "unit", + "value": "watt" + }, + { + "id": "decimals", + "value": 3 + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge", + "valueDisplayMode": "text" + } + }, + { + "id": "color", + "value": { + "mode": "continuous-GrYlRd" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Activity (F/s)" + }, + "properties": [ + { + "id": "unit", + "value": " F/s" + }, + { + "id": "decimals", + "value": 3 + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge", + "valueDisplayMode": "text" + } + }, + { + "id": "color", + "value": { + "mode": "continuous-GrYlRd" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Energy (kJ, window)" + }, + "properties": [ + { + "id": "decimals", + "value": 1 + }, + { + "id": "unit", + "value": "kJ" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge", + "valueDisplayMode": "text" + } + }, + { + "id": "color", + "value": { + "mode": "continuous-BlPu" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 8, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": [ + "Power (W)", + "Activity (F/s)", + "Energy (kJ)" + ], + "reducer": [ + "sum" + ], + "show": true + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Power (W)" + } + ] + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod, namespace) (rate({__name__=\"perf.core.energy_joules_total\", \"resctrl.group.source\"=\"pod\"}[$__rate_interval]) * on(\"k8s.pod.uid\") group_left(pod, namespace) label_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info[$__range])), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\"))", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "power", + "interval": "5s" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod, namespace) (rate({__name__=\"perf.activity_farads_total\", \"resctrl.group.source\"=\"pod\"}[$__rate_interval]) * on(\"k8s.pod.uid\") group_left(pod, namespace) label_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info[$__range])), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\"))", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "activity", + "interval": "5s" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod, namespace) (increase({__name__=\"perf.core.energy_joules_total\", \"resctrl.group.source\"=\"pod\"}[$__range]) * on(\"k8s.pod.uid\") group_left(pod, namespace) label_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info[$__range])), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\")) / 1000", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "energy" + } + ], + "title": "Pod Energy & Activity Table", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "renameByName": { + "Value #activity": "Activity (F/s)", + "Value #energy": "Energy (kJ, window)", + "Value #power": "Power (W)", + "namespace": "Namespace", + "pod": "Pod" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 104, + "title": "Package Totals", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total power per CPU package (domain) including unmonitored background workloads", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Power (W)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (\"domain.id\") (rate({__name__=\"perf.core.energy_joules_total\"}[$__rate_interval]))", + "legendFormat": "{{domain}}", + "refId": "A", + "interval": "5s" + } + ], + "title": "Package Total Power (all workloads)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total activity per package (domain) in F/s \u2014 includes all workloads on the system", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Activity (F/s)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": " F/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (\"domain.id\") (rate({__name__=\"perf.activity_farads_total\"}[$__rate_interval]))", + "legendFormat": "{{domain}}", + "refId": "A", + "interval": "5s" + } + ], + "title": "Package Total Activity", + "type": "timeseries" + } + ], + "refresh": "15s", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "resctrl", + "aet", + "energy", + "kubernetes" + ], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Prometheus", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(kube_pod_info, namespace)", + "hide": 0, + "includeAll": true, + "label": "Namespace", + "multi": true, + "name": "namespace", + "query": { + "query": "label_values(kube_pod_info, namespace)", + "refId": "ns" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Kubernetes Pod Energy (Intel AET / resctrl-mon)", + "uid": "resctrl-pod-energy" +} \ No newline at end of file diff --git a/deployment/helm/resctrl-mon/optional/otel-collector-agent.yaml b/deployment/helm/resctrl-mon/optional/otel-collector-agent.yaml new file mode 100644 index 000000000..6a70b4c15 --- /dev/null +++ b/deployment/helm/resctrl-mon/optional/otel-collector-agent.yaml @@ -0,0 +1,92 @@ +# Optional OTel Collector agent DaemonSet for resctrl-mon OTLP push path. +# +# Deploy this alongside nri-resctrl-mon when telemetry.otlp.enabled=true. +# It receives OTLP from the plugin, enriches with k8sattributes, and +# fans out to Prometheus and/or other backends. +# +# Prerequisites: +# - otel-collector-rbac.yaml (ServiceAccount + ClusterRole for pod metadata) +# +# Usage: +# kubectl apply -f otel-collector-rbac.yaml +# kubectl apply -f otel-collector-agent.yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: otel-collector-resctrl-config + namespace: monitoring +data: + config.yaml: | + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + processors: + k8sattributes: + auth_type: serviceAccount + extract: + metadata: + - k8s.namespace.name + - k8s.pod.name + - k8s.node.name + pod_association: + - sources: + - from: resource_attribute + name: k8s.pod.uid + batch: + timeout: 10s + exporters: + prometheus: + endpoint: 0.0.0.0:8889 + resource_to_telemetry_conversion: + enabled: true + service: + pipelines: + metrics: + receivers: [otlp] + processors: [k8sattributes, batch] + exporters: [prometheus] +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: otel-collector-resctrl + namespace: monitoring + labels: + app.kubernetes.io/name: otel-collector-resctrl +spec: + selector: + matchLabels: + app.kubernetes.io/name: otel-collector-resctrl + template: + metadata: + labels: + app.kubernetes.io/name: otel-collector-resctrl + spec: + serviceAccountName: otel-collector-resctrl + containers: + - name: collector + image: otel/opentelemetry-collector-contrib:0.104.0 + args: ["--config=/etc/otel/config.yaml"] + ports: + - name: otlp-grpc + containerPort: 4317 + protocol: TCP + - name: prom-export + containerPort: 8889 + protocol: TCP + volumeMounts: + - name: config + mountPath: /etc/otel + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + memory: 256Mi + volumes: + - name: config + configMap: + name: otel-collector-resctrl-config diff --git a/deployment/helm/resctrl-mon/optional/otel-collector-rbac.yaml b/deployment/helm/resctrl-mon/optional/otel-collector-rbac.yaml new file mode 100644 index 000000000..0114a49ad --- /dev/null +++ b/deployment/helm/resctrl-mon/optional/otel-collector-rbac.yaml @@ -0,0 +1,33 @@ +# RBAC for the OTel Collector k8sattributes processor. +# Grants read access to pod metadata so the processor can enrich metrics. +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: otel-collector-resctrl + namespace: monitoring +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: otel-collector-resctrl +rules: + - apiGroups: [""] + resources: ["pods", "namespaces", "nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: ["replicasets"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: otel-collector-resctrl +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: otel-collector-resctrl +subjects: + - kind: ServiceAccount + name: otel-collector-resctrl + namespace: monitoring diff --git a/deployment/helm/resctrl-mon/templates/configmap.yaml b/deployment/helm/resctrl-mon/templates/configmap.yaml index 75562a9bf..c64d426c9 100644 --- a/deployment/helm/resctrl-mon/templates/configmap.yaml +++ b/deployment/helm/resctrl-mon/templates/configmap.yaml @@ -10,3 +10,28 @@ data: resctrlPath: {{ .Values.resctrlPath }} namespaces: [] labelSelector: {} + telemetry: + prometheus: + enabled: {{ .Values.telemetry.prometheus.enabled }} + listenAddress: {{ .Values.telemetry.prometheus.listenAddress | quote }} + namespace: {{ .Values.telemetry.prometheus.namespace | quote }} + otlp: + enabled: {{ .Values.telemetry.otlp.enabled }} + endpoint: {{ .Values.telemetry.otlp.endpoint | quote }} + protocol: {{ .Values.telemetry.otlp.protocol | quote }} + interval: {{ .Values.telemetry.otlp.interval | quote }} + insecure: {{ .Values.telemetry.otlp.insecure }} + perfCounters: + enabled: {{ .Values.telemetry.perfCounters.enabled }} + {{- with .Values.telemetry.perfCounters.include }} + include: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.telemetry.perfCounters.exclude }} + exclude: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.telemetry.resourceAttributes }} + resourceAttributes: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/deployment/helm/resctrl-mon/templates/daemonset.yaml b/deployment/helm/resctrl-mon/templates/daemonset.yaml index 629a52dda..9293451d9 100644 --- a/deployment/helm/resctrl-mon/templates/daemonset.yaml +++ b/deployment/helm/resctrl-mon/templates/daemonset.yaml @@ -13,6 +13,13 @@ spec: metadata: labels: {{- include "nri-plugin.labels" . | nindent 8 }} + {{- if .Values.telemetry.prometheus.enabled }} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "{{ (split ":" .Values.telemetry.prometheus.listenAddress)._1 | default "9100" }}" + prometheus.io/path: "/metrics" + prometheus.io/interval: "{{ .Values.telemetry.prometheus.scrapeInterval }}" + {{- end }} spec: {{- with .Values.tolerations }} tolerations: @@ -61,6 +68,12 @@ spec: - -v image: {{ .Values.image.name }}:{{ .Values.image.tag | default .Chart.AppVersion }} imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.telemetry.prometheus.enabled }} + ports: + - name: metrics + containerPort: {{ (split ":" .Values.telemetry.prometheus.listenAddress)._1 | default "9100" | int }} + protocol: TCP + {{- end }} resources: requests: cpu: {{ .Values.resources.cpu }} diff --git a/deployment/helm/resctrl-mon/values.yaml b/deployment/helm/resctrl-mon/values.yaml index 26abf0cd0..e38c43d20 100644 --- a/deployment/helm/resctrl-mon/values.yaml +++ b/deployment/helm/resctrl-mon/values.yaml @@ -4,6 +4,25 @@ --- resctrlPath: /sys/fs/resctrl +# Telemetry configuration — embedded OTel exporter. +telemetry: + prometheus: + enabled: true + listenAddress: ":9100" + scrapeInterval: "15s" # recommended scrape interval (annotation hint) + namespace: "" # empty = resctrl_* Prometheus metric names + otlp: + enabled: false + endpoint: "" # e.g. "otel-collector.monitoring.svc:4317" + protocol: grpc # grpc | http + interval: 15s + insecure: true + perfCounters: + enabled: false # gate rdt=perf counters (c1_res, stalls_*, etc.) + include: [] + exclude: [] + resourceAttributes: {} # static OTel resource attributes + image: name: ghcr.io/containers/nri-plugins/nri-resctrl-mon # tag, if defined will use the given image tag, otherwise Chart.AppVersion will be used diff --git a/go.mod b/go.mod index eebdbe85b..067f21b6d 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,6 @@ require ( github.com/containers/nri-plugins/pkg/topology v0.0.0 github.com/coreos/go-systemd/v22 v22.7.0 github.com/fsnotify/fsnotify v1.10.1 - github.com/google/uuid v1.6.0 github.com/intel/goresctrl v0.13.0 github.com/intel/memtierd v0.1.1 github.com/k8stopologyawareschedwg/noderesourcetopology-api v0.1.3 @@ -67,6 +66,7 @@ require ( github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -114,6 +114,9 @@ require ( replace ( github.com/containers/nri-plugins/pkg/topology v0.0.0 => ./pkg/topology + // Temporary: depends on unreleased goresctrl pkg/monitor (intel/goresctrl#192). + // Drop once a goresctrl release containing pkg/monitor is available. + github.com/intel/goresctrl v0.13.0 => github.com/cmcantalupo/goresctrl v0.0.0-20260812225045-3705888589be github.com/opencontainers/runtime-tools => github.com/opencontainers/runtime-tools v0.0.0-20221026201742-946c877fa809 ) diff --git a/go.sum b/go.sum index 19e7836e8..985dfef06 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1x github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cmcantalupo/goresctrl v0.0.0-20260812225045-3705888589be h1:/U5tRh81tg8WRT1eumPNo7AUP/p0PT3MsXEhRunRssI= +github.com/cmcantalupo/goresctrl v0.0.0-20260812225045-3705888589be/go.mod h1:tCyfuJ95wo5HR0SI2TybDiew+BN5wocABcaX8N7ZAQg= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/nri v0.12.2 h1:piA7h2QUm0m3+e994qWFPYwnoXXwAphwDc1Ydx1eWi4= @@ -71,8 +73,6 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= -github.com/intel/goresctrl v0.13.0 h1:5fhKjNq4V5MYDFHa//6M6x0jP6Iq5EXwZc6/eYxdEtQ= -github.com/intel/goresctrl v0.13.0/go.mod h1:KFHS91JGOmeeuEog+nTQcsGjLC81nRqdsdhcqf69fjU= github.com/intel/memtierd v0.1.1 h1:hGSN0+dzjaUkwgkJrk6B9SU4dntggXLpXgs9Dm+jfz4= github.com/intel/memtierd v0.1.1/go.mod h1:NFDBvjoDS42gBK/c9q/CYCJ2pt/+g7UQwOOBvQli4z0= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= diff --git a/sample-configs/nri-resctrl-mon.yaml b/sample-configs/nri-resctrl-mon.yaml index 9cbd339a0..a3b649e9e 100644 --- a/sample-configs/nri-resctrl-mon.yaml +++ b/sample-configs/nri-resctrl-mon.yaml @@ -1,3 +1,19 @@ resctrlPath: /sys/fs/resctrl namespaces: [] labelSelector: {} +telemetry: + prometheus: + enabled: true + listenAddress: ":9100" + namespace: "" + otlp: + enabled: false + endpoint: "" + protocol: grpc + interval: 15s + insecure: true + perfCounters: + enabled: false + include: [] + exclude: [] + resourceAttributes: {} From ce27891efbbf93af8961925dcc66eba678543342 Mon Sep 17 00:00:00 2001 From: "Christopher M. Cantalupo" Date: Mon, 24 Aug 2026 11:54:42 -0700 Subject: [PATCH 2/2] resctrl-mon: address review feedback on telemetry plugin and deployment Fold the review-round fixes for the resctrl-mon plugin, its tests, and the reference deployment assets into a single change on top of the initial OTel-telemetry feature. Plugin lifecycle and config reload: - Start telemetry from Configure() once a configuration is available rather than before the plugin is configured, and shut the previous telemetry down under a bounded timeout on reload. - Treat the resctrl root as immutable once the plugin is running: reject a setConfig that changes resctrlPath after telemetry/reconciliation start instead of swapping in an empty manager (which would drop the in-memory tracking and PIDs for every live pod, skip re-synchronization, and let a pending removal bound to the old manager delete a same-UID group in the new one). The initial configuration (from a --config file and/or the NRI server, applied before the plugin runs) may still select a non-default root and rebuild the manager; a restart is required to change it later. - Fix the reconciler lifecycle: capture the stop channel and manager in locals, and restart the reconciler only when the manager is (re)created. - Publish the telemetry state only after instrument registration succeeds. - Make a telemetry/filter reload transactional: if restarting telemetry fails (e.g. the new Prometheus port is occupied), roll back to the previous config, manager, telemetry, and reconciler instead of leaving them permanently disabled. - Retry leaked mon_groups: track keys whose Remove failed in a pending set and retry them from the reconciler so a failed rmdir is not lost. Synchronize and metrics: - Seed the reconcile live set from the monitored pod sandboxes, not only from containers, so a sandbox that is alive with no running container (e.g. between container restarts) keeps its mon_group. Remember that live set so the background reconciler keeps protecting a container-less sandbox (which is never passed to EnsureGroup and so never appears in Manager.List()); drop a key from it in RemovePodSandbox so a truly gone sandbox can still be reaped. - Replace the per-call regex glob with a simple wildcard matcher. - Derive the resctrl control group relative to the configured root so a non-default root (e.g. /mnt/rdt) is handled correctly. - The manager validates and tracks pod UIDs only, so fix the resctrl.group.source attribute to the constant "pod" and drop the unreachable non-pod ("other") classification. - Default otlp.insecure to true so the runtime matches the chart, sample config, and README, which all document plaintext OTLP by default. Tests: - Harden the existing tests, bind the telemetry test to an ephemeral port, table-drive controlGroupOf, and add coverage for orphan mon_group removal, pending-removal retry, rejected resctrl root changes on a running plugin, accepted initial root selection, and reconciler preservation of a container-less live sandbox. Deployment assets: - Correct the helm namespace documentation and values, and fix the pod legend in the energy dashboard. - Translate the dashboard metric and label names to the names the OTel Prometheus exporter actually emits (e.g. perf_core_energy_joules_total, k8s_pod_uid), and wire the DS_PROMETHEUS and namespace template variables through both dashboards so they import against any Prometheus datasource. - Relabel the misleading "all workloads" package panels to reflect that only per-pod mon_groups are observed, fix the domain legend, and filter the package-total panels on resctrl_group_source="pod". - otel-collector: promote the k8s.pod.uid data-point attribute to resource scope with groupbyattrs so k8sattributes can associate it, add the Service that exposes the documented OTLP endpoint, create the monitoring namespace so the reference manifests apply on a clean cluster, and correct the endpoint examples. - README: mark the AET perf/energy counters (rdt=perf) as pending upstream rather than available in a released kernel. - Run install-plugins under 'set -e' so a failed install aborts the target. Signed-off-by: Christopher M. Cantalupo --- Makefile | 3 +- cmd/plugins/resctrl-mon/main.go | 5 - cmd/plugins/resctrl-mon/metrics.go | 77 +++--- cmd/plugins/resctrl-mon/plugin.go | 242 +++++++++++++++--- cmd/plugins/resctrl-mon/plugin_test.go | 149 ++++++++++- cmd/plugins/resctrl-mon/telemetry.go | 9 +- cmd/plugins/resctrl-mon/telemetry_test.go | 36 +-- deployment/helm/resctrl-mon/README.md | 12 +- .../grafana-resctrl-perf-counters.json | 98 ++++--- .../optional/grafana-resctrl-pod-energy.json | 90 +++---- .../optional/otel-collector-agent.yaml | 36 ++- .../optional/otel-collector-rbac.yaml | 8 + deployment/helm/resctrl-mon/values.yaml | 4 +- 13 files changed, 564 insertions(+), 205 deletions(-) diff --git a/Makefile b/Makefile index dbafb92ba..736835a66 100644 --- a/Makefile +++ b/Makefile @@ -165,7 +165,8 @@ NRI_DEFAULT_IDX ?= 90 # install-plugins: install plugin binaries to NRI_PLUGINS_DIR with the # - prefix required by CRI-O/containerd static plugin discovery. install-plugins: build-plugins - $(Q)echo "Installing plugins to $(NRI_PLUGINS_DIR)..."; \ + $(Q)set -e; \ + echo "Installing plugins to $(NRI_PLUGINS_DIR)..."; \ mkdir -p $(NRI_PLUGINS_DIR); \ for bin in $(PLUGINS); do \ name=$${bin#nri-}; \ diff --git a/cmd/plugins/resctrl-mon/main.go b/cmd/plugins/resctrl-mon/main.go index fc6eb5f85..eedf56599 100644 --- a/cmd/plugins/resctrl-mon/main.go +++ b/cmd/plugins/resctrl-mon/main.go @@ -69,11 +69,6 @@ func main() { } } - // Start OTel telemetry (Prometheus endpoint + optional OTLP push). - if err := p.startTelemetry(context.Background()); err != nil { - log.Fatalf("failed to start telemetry: %v", err) - } - opts := []stub.Option{ stub.WithOnClose(p.onClose), } diff --git a/cmd/plugins/resctrl-mon/metrics.go b/cmd/plugins/resctrl-mon/metrics.go index d44ba4ed3..d919bd7b9 100644 --- a/cmd/plugins/resctrl-mon/metrics.go +++ b/cmd/plugins/resctrl-mon/metrics.go @@ -16,7 +16,6 @@ package main import ( "path/filepath" - "regexp" "strings" "github.com/intel/goresctrl/pkg/monitor" @@ -32,10 +31,10 @@ var coreAETFiles = map[string]bool{ } // setupMetrics registers OTel instruments via the goresctrl adapter. -func setupMetrics(mgr *monitor.Manager, cfg telemetryConfig, meter otelmetric.Meter) (*monitor.Registration, error) { +func setupMetrics(mgr *monitor.Manager, cfg telemetryConfig, resctrlRoot string, meter otelmetric.Meter) (*monitor.Registration, error) { return mgr.RegisterOTelInstruments(meter, monitor.WithFilter(perfCounterFilter(cfg)), - monitor.WithAttributes(groupAttributes), + monitor.WithAttributes(groupAttributesFor(resctrlRoot)), ) } @@ -75,47 +74,57 @@ func perfCounterFilter(cfg telemetryConfig) monitor.FilterFunc { } } -// groupAttributes provides per-group OTel attributes. -func groupAttributes(key, path string) []attribute.KeyValue { - return []attribute.KeyValue{ - attribute.String("k8s.pod.uid", key), - attribute.String("resctrl.control_group", controlGroupOf(path)), - attribute.String("resctrl.group.source", sourceFor(key)), +// groupAttributesFor returns a per-group OTel attribute function bound to the +// configured resctrl root, which is needed to recognize the root ctrl_group. +func groupAttributesFor(resctrlRoot string) monitor.AttributeFunc { + root := filepath.Clean(resctrlRoot) + return func(key, path string) []attribute.KeyValue { + return []attribute.KeyValue{ + attribute.String("k8s.pod.uid", key), + attribute.String("resctrl.control_group", controlGroupOf(root, path)), + // The manager validates and tracks pod UIDs only, so every exported + // group is pod-sourced. + attribute.String("resctrl.group.source", "pod"), + } } } // matchGlob does simple glob matching (only * is supported as wildcard). func matchGlob(pattern, name string) bool { - if pattern == name { - return true + if !strings.Contains(pattern, "*") { + return pattern == name + } + parts := strings.Split(pattern, "*") + // The name must start with the segment before the first '*' and end with + // the segment after the last '*'. + if !strings.HasPrefix(name, parts[0]) { + return false } - if strings.Contains(pattern, "*") { - re := "^" + strings.ReplaceAll(regexp.QuoteMeta(pattern), `\*`, ".*") + "$" - matched, _ := regexp.MatchString(re, name) - return matched + name = name[len(parts[0]):] + last := parts[len(parts)-1] + if !strings.HasSuffix(name, last) { + return false + } + name = name[:len(name)-len(last)] + // Any interior segments must appear in order. + for _, seg := range parts[1 : len(parts)-1] { + i := strings.Index(name, seg) + if i < 0 { + return false + } + name = name[i+len(seg):] } - return false + return true } -// controlGroupOf extracts the CTRL group name from a mon_group path. -// e.g. "/sys/fs/resctrl/COS1/mon_groups/abc-123" → "COS1" -// e.g. "/sys/fs/resctrl/mon_groups/abc-123" → "" (root/default) -func controlGroupOf(groupPath string) string { +// controlGroupOf extracts the CTRL group name from a mon_group path, relative +// to the configured resctrl root. +// e.g. root=/sys/fs/resctrl, "/sys/fs/resctrl/COS1/mon_groups/abc-123" → "COS1" +// e.g. root=/sys/fs/resctrl, "/sys/fs/resctrl/mon_groups/abc-123" → "" (root) +func controlGroupOf(resctrlRoot, groupPath string) string { ctrlDir := filepath.Dir(filepath.Dir(groupPath)) - base := filepath.Base(ctrlDir) - if base == "resctrl" || base == "." || base == "/" { + if filepath.Clean(ctrlDir) == filepath.Clean(resctrlRoot) { return "" } - return base -} - -// uuidPattern matches standard dashed UUID (pod UIDs). -var uuidPattern = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) - -// sourceFor classifies a group key as "pod" (UUID) or "other". -func sourceFor(key string) string { - if uuidPattern.MatchString(key) { - return "pod" - } - return "other" + return filepath.Base(ctrlDir) } diff --git a/cmd/plugins/resctrl-mon/plugin.go b/cmd/plugins/resctrl-mon/plugin.go index 1828fd477..36af6db48 100644 --- a/cmd/plugins/resctrl-mon/plugin.go +++ b/cmd/plugins/resctrl-mon/plugin.go @@ -23,6 +23,7 @@ import ( "slices" "strconv" "strings" + "sync" "time" "sigs.k8s.io/yaml" @@ -50,6 +51,10 @@ type plugin struct { stopReconciler chan struct{} // closed to stop the background reconciler telemetry *telemetryState metrics *monitor.Registration + + mu sync.Mutex // guards pendingRemoval and liveKeys + pendingRemoval map[string]struct{} // keys whose Remove failed, retried by the reconciler + liveKeys map[string]struct{} // monitored pod sandboxes alive as of the last Synchronize } // pluginConfig holds the runtime configuration for the plugin. @@ -102,6 +107,15 @@ func (p *plugin) Configure(ctx context.Context, config, runtime, version string) return 0, err } } + // Start telemetry now that configuration (from a --config file and/or the + // NRI server) is finalized. Binding here rather than in main() lets a + // runtime-provided config disable Prometheus or pick a different port + // before we bind, instead of fatally exiting on a pre-config port clash. + if p.telemetry == nil { + if err := p.startTelemetry(ctx); err != nil { + return 0, err + } + } return 0, nil } @@ -142,49 +156,98 @@ func (p *plugin) setConfig(data []byte) error { return fmt.Errorf("setConfig: %w", err) } - // Recreate the monitor manager with the new path before mutating any plugin - // state, so a failure here leaves the running configuration fully intact. - mgr, err := monitor.New(monitor.Options{ - ResctrlRoot: cfg.ResctrlPath, - KeyValidator: monitor.PodUIDValidator, - KeyCanonicalizer: monitor.CanonicalizePodUID, - }) - if err != nil { - return fmt.Errorf("setConfig: failed to create monitor manager: %w", err) + // The resctrl root cannot be changed once the plugin is running: swapping + // the manager would drop the in-memory tracking (and assigned PIDs) for + // every live pod with no re-synchronization until the next lifecycle event, + // and a pending removal bound to the old manager could later delete a + // same-UID group created in the new one. The initial configuration is + // applied (from a --config file and/or the NRI server) before telemetry and + // the reconciler start, so it may still select a non-default root and + // rebuild the manager; only reject the change once the plugin is running. + running := p.telemetry != nil || p.stopReconciler != nil + if running && cfg.ResctrlPath != p.config.ResctrlPath { + return fmt.Errorf("setConfig: resctrlPath cannot be changed on a running plugin (have %q, got %q); restart the plugin to change it", + p.config.ResctrlPath, cfg.ResctrlPath) + } + + // Create the monitor manager on the initial configuration only (the root is + // immutable thereafter, per the guard above). The manager holds the + // in-memory tracking for every running pod. Build it before mutating state so + // a failure here leaves the running configuration fully intact. + rootChanged := p.config == nil || cfg.ResctrlPath != p.config.ResctrlPath + var newMgr *monitor.Manager + if rootChanged { + var err error + newMgr, err = monitor.New(monitor.Options{ + ResctrlRoot: cfg.ResctrlPath, + KeyValidator: monitor.PodUIDValidator, + KeyCanonicalizer: monitor.CanonicalizePodUID, + }) + if err != nil { + return fmt.Errorf("setConfig: failed to create monitor manager: %w", err) + } } - p.config = &cfg + // Remember the currently-applied config/manager so a failed reload can be + // rolled back to the last working state instead of leaving telemetry and + // the reconciler permanently disabled. + prevConfig := p.config + prevMgr := p.mgr - // Stop the old reconciler before swapping the manager; it holds a - // reference to the old manager and would otherwise leak. - if p.stopReconciler != nil { - close(p.stopReconciler) - p.stopReconciler = nil - } + p.config = &cfg - // If telemetry is already running, its OTel batch callback is bound to the - // manager we are about to replace (which is pinned to the previous - // ResctrlPath). Tear it down so we neither leak the old manager nor keep a - // second registration reading the old root, then restart it against the new - // manager below. + // If telemetry is already running, tear it down so it can be rebound to the + // (possibly new) manager with the new telemetry settings below. restartTelemetry := p.telemetry != nil if restartTelemetry { if p.metrics != nil { _ = p.metrics.Unregister() p.metrics = nil } - p.telemetry.shutdown(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), telemetryShutdownTimeout) + p.telemetry.shutdown(ctx) + cancel() p.telemetry = nil } - p.mgr = mgr + // Swap the manager only on a root change. The background reconciler pins the + // manager it was started with, so stop it here and restart it below against + // the new manager; on an unchanged root it keeps running untouched. + reconcilerWasRunning := p.stopReconciler != nil + if rootChanged { + if p.stopReconciler != nil { + close(p.stopReconciler) + p.stopReconciler = nil + } + p.mgr = newMgr + } if restartTelemetry { if err := p.startTelemetry(context.Background()); err != nil { + // Roll back to the previously working configuration: the old + // telemetry/manager have already been torn down, so restore the + // prior config/manager and bring telemetry (and, if we swapped the + // manager, the reconciler) back up. This keeps a failed reload + // (e.g. the new Prometheus port is occupied) from permanently + // disabling telemetry and reconciliation. + p.config = prevConfig + if rootChanged { + p.mgr = prevMgr + } + if rerr := p.startTelemetry(context.Background()); rerr != nil { + log.Errorf("setConfig: failed to restore telemetry after failed reload: %v", rerr) + } + if rootChanged && reconcilerWasRunning { + p.startReconciler() + } return fmt.Errorf("setConfig: restart telemetry: %w", err) } } + if rootChanged && reconcilerWasRunning { + p.startReconciler() + } + log.Debugf("configuration: resctrlPath=%s namespaces=%v labelSelector=%v", cfg.ResctrlPath, cfg.Namespaces, cfg.LabelSelector) return nil @@ -202,9 +265,19 @@ func (p *plugin) Synchronize(ctx context.Context, pods []*api.PodSandbox, contai podBySandboxID[pod.GetId()] = pod } - // Create mon_groups for running containers that don't have one, - // and write their PIDs to ensure monitoring is active after restart. - var liveKeys []string + // A pod sandbox can be alive with no running container (for example between + // container restarts). Seed the reconcile live set from the monitored + // sandboxes so their existing mon_groups survive; the container loop below + // then creates missing groups and (re)assigns PIDs. Remember this set so the + // background reconciler keeps protecting container-less sandboxes (which are + // never passed to EnsureGroup and so never appear in mgr.List()). + liveKeys := make([]string, 0, len(pods)) + for _, pod := range pods { + if p.shouldMonitorPod(pod) { + liveKeys = append(liveKeys, pod.GetUid()) + } + } + p.setLiveKeys(liveKeys) for _, ctr := range containers { pod, ok := podBySandboxID[ctr.GetPodSandboxId()] if !ok { @@ -222,7 +295,6 @@ func (p *plugin) Synchronize(ctx context.Context, pods []*api.PodSandbox, contai log.Warnf("Synchronize: failed to create mon_group for pod %s: %v", podUID, err) continue } - liveKeys = append(liveKeys, podUID) pid := int(ctr.GetPid()) if pid > 0 { @@ -247,32 +319,125 @@ func (p *plugin) Synchronize(ctx context.Context, pods []*api.PodSandbox, contai return nil, nil } -// startReconciler launches a background goroutine that periodically removes -// orphaned mon_group directories. This handles the case where removeMonGroup -// fails in StopContainer (e.g., kernel busy) and the directory lingers. +// startReconciler launches a background goroutine that periodically retries +// failed removals and removes orphaned mon_group directories. This handles the +// case where removeMonGroup fails in RemovePodSandbox (e.g., kernel busy) and +// the directory lingers. func (p *plugin) startReconciler() { if p.stopReconciler != nil { // Already running from a previous Synchronize call. return } - p.stopReconciler = make(chan struct{}) + // Capture the channel and manager this goroutine owns. A later config reload + // may close p.stopReconciler, swap p.mgr, and start a fresh goroutine; binding + // to these locals keeps this goroutine's select on an immutable channel and + // pinned to the manager it was started with. + stop := make(chan struct{}) + p.stopReconciler = stop + mgr := p.mgr go func() { ticker := time.NewTicker(reconcileInterval) defer ticker.Stop() for { select { - case <-p.stopReconciler: + case <-stop: return case <-ticker.C: - if err := p.mgr.Reconcile(p.mgr.List()); err != nil { - log.Warnf("reconciler: %v", err) - } + p.reconcile(mgr) } } }() log.Debugf("background reconciler started (interval=%s)", reconcileInterval) } +// reconcile retries removals that previously failed and reaps untracked orphan +// directories. A failed Remove leaves its key tracked, so Reconcile(List()) +// would treat it as live forever; retrying Remove is what actually frees the +// RMID once the kernel releases the directory. +func (p *plugin) reconcile(mgr *monitor.Manager) { + for _, key := range p.pendingRemovalKeys() { + switch err := mgr.Remove(key); { + case err == nil, errors.Is(err, monitor.ErrNotTracked): + p.clearPendingRemoval(key) + default: + log.Warnf("reconciler: retry remove %s failed: %v", key, err) + } + } + // Reconcile against the tracked keys plus the live sandbox set: a + // container-less sandbox is protected only by liveKeys (it is never passed + // to EnsureGroup, so mgr.List() omits it), and reconciling without it would + // reap its existing mon_group and hand its replacement container a fresh + // RMID. + if err := mgr.Reconcile(p.reconcileLiveSet(mgr)); err != nil { + log.Warnf("reconciler: %v", err) + } +} + +// setLiveKeys records the set of monitored pod sandboxes that are alive, so the +// background reconciler can protect their mon_groups even when no container has +// caused them to be tracked in the Manager. +func (p *plugin) setLiveKeys(keys []string) { + p.mu.Lock() + defer p.mu.Unlock() + p.liveKeys = make(map[string]struct{}, len(keys)) + for _, k := range keys { + p.liveKeys[k] = struct{}{} + } +} + +// dropLiveKey removes a sandbox key from the live set once its pod is gone, so +// the reconciler stops protecting it and can reap any leftover mon_group. +func (p *plugin) dropLiveKey(key string) { + p.mu.Lock() + defer p.mu.Unlock() + delete(p.liveKeys, key) +} + +// reconcileLiveSet returns the union of the Manager's tracked keys and the live +// sandbox set for use as the reconcile live list. +func (p *plugin) reconcileLiveSet(mgr *monitor.Manager) []string { + p.mu.Lock() + defer p.mu.Unlock() + live := make(map[string]struct{}, len(p.liveKeys)) + for k := range p.liveKeys { + live[k] = struct{}{} + } + for _, k := range mgr.List() { + live[k] = struct{}{} + } + keys := make([]string, 0, len(live)) + for k := range live { + keys = append(keys, k) + } + return keys +} + +// markPendingRemoval records a key whose Remove failed so the reconciler retries it. +func (p *plugin) markPendingRemoval(key string) { + p.mu.Lock() + defer p.mu.Unlock() + if p.pendingRemoval == nil { + p.pendingRemoval = make(map[string]struct{}) + } + p.pendingRemoval[key] = struct{}{} +} + +func (p *plugin) clearPendingRemoval(key string) { + p.mu.Lock() + defer p.mu.Unlock() + delete(p.pendingRemoval, key) +} + +func (p *plugin) pendingRemovalKeys() []string { + p.mu.Lock() + defer p.mu.Unlock() + keys := make([]string, 0, len(p.pendingRemoval)) + for k := range p.pendingRemoval { + keys = append(keys, k) + } + return keys +} + // PostCreateContainer is called after the container is created but before // it starts executing. The container PID is NOT yet available (pid=0) because // the init process has not been started. We create the mon_group here so it @@ -371,6 +536,10 @@ func (p *plugin) PostStartContainer(ctx context.Context, pod *api.PodSandbox, ct func (p *plugin) RemovePodSandbox(ctx context.Context, pod *api.PodSandbox) error { podUID := pod.GetUid() + // The sandbox is gone, so stop protecting its key in the reconciler's live + // set; otherwise a failed Remove below could never be reaped. + p.dropLiveKey(podUID) + // Attempt removal unconditionally rather than gating on shouldMonitorPod: a // pod may have been monitored under a configuration that was later changed to // exclude it. Gating here would strand its mon_group, because Remove would @@ -383,8 +552,9 @@ func (p *plugin) RemovePodSandbox(ctx context.Context, pod *api.PodSandbox) erro case errors.Is(err, monitor.ErrNotTracked): // Pod was never monitored; nothing to clean up. default: - log.Warnf("RemovePodSandbox %s/%s: failed to remove mon_group (will be cleaned by reconciler): %v", + log.Warnf("RemovePodSandbox %s/%s: failed to remove mon_group (will be retried by reconciler): %v", pod.GetNamespace(), pod.GetName(), err) + p.markPendingRemoval(podUID) } return nil } diff --git a/cmd/plugins/resctrl-mon/plugin_test.go b/cmd/plugins/resctrl-mon/plugin_test.go index da7479bc5..60507f894 100644 --- a/cmd/plugins/resctrl-mon/plugin_test.go +++ b/cmd/plugins/resctrl-mon/plugin_test.go @@ -37,8 +37,9 @@ func newTestPlugin(resctrlPath string) *plugin { ResctrlPath: resctrlPath, } mgr, err := monitor.New(monitor.Options{ - ResctrlRoot: resctrlPath, - KeyValidator: monitor.PodUIDValidator, + ResctrlRoot: resctrlPath, + KeyValidator: monitor.PodUIDValidator, + KeyCanonicalizer: monitor.CanonicalizePodUID, }) if err != nil { panic(err) @@ -207,7 +208,7 @@ func TestSetConfig(t *testing.T) { p := newTestPlugin("/tmp/resctrl-test") configYAML := []byte(` -resctrlPath: /tmp/test-resctrl +resctrlPath: /tmp/resctrl-test namespaces: - production - staging @@ -217,7 +218,7 @@ labelSelector: err := p.setConfig(configYAML) require.NoError(t, err) - assert.Equal(t, "/tmp/test-resctrl", p.config.ResctrlPath) + assert.Equal(t, "/tmp/resctrl-test", p.config.ResctrlPath) assert.Equal(t, []string{"production", "staging"}, p.config.Namespaces) assert.Equal(t, map[string]string{"monitor": "true"}, p.config.LabelSelector) } @@ -260,6 +261,91 @@ func TestSynchronize_UsesUIDNotSandboxID(t *testing.T) { assert.NoError(t, err) } +func TestSynchronize_RemovesOrphanMonGroup(t *testing.T) { + tmpDir := t.TempDir() + + // An orphaned mon_group left behind by a previous run, keyed by a + // UUID-shaped pod UID that is no longer live. + orphanUID := "deadbeef-0000-4000-8000-000000000000" + orphanDir := filepath.Join(tmpDir, "mon_groups", orphanUID) + require.NoError(t, os.MkdirAll(orphanDir, 0755)) + + p := newTestPlugin(tmpDir) + + // Synchronize with a single live pod that is not the orphan. + podUID := "a1b2c3d4-e5f6-7890-abcd-ef1234567890" + pod := makePod(podUID, "default", "live-pod") + ctr := makeContainer("c1", "container1", pod.GetId(), 0, "") + + _, err := p.Synchronize(context.Background(), []*api.PodSandbox{pod}, []*api.Container{ctr}) + require.NoError(t, err) + + // The live pod's mon_group exists... + _, err = os.Stat(filepath.Join(tmpDir, "mon_groups", podUID)) + assert.NoError(t, err) + + // ...and the orphan was reaped via Reconcile. + _, err = os.Stat(orphanDir) + assert.True(t, os.IsNotExist(err), "orphan mon_group should have been removed by Reconcile") +} + +// TestReconcile_PreservesContainerlessLiveSandbox verifies that a monitored pod +// sandbox that is alive with no running container (its mon_group survives from +// a previous run but no EnsureGroup tracks it) is not reaped by the background +// reconciler, so a restarting container reuses the same RMID. +func TestReconcile_PreservesContainerlessLiveSandbox(t *testing.T) { + tmpDir := t.TempDir() + + // A container-less sandbox: its mon_group exists on disk but the plugin + // never calls EnsureGroup for it, so mgr.List() will not include it. + sandboxUID := "a1b2c3d4-e5f6-7890-abcd-ef1234567890" + sandboxDir := filepath.Join(tmpDir, "mon_groups", sandboxUID) + require.NoError(t, os.MkdirAll(sandboxDir, 0755)) + + p := newTestPlugin(tmpDir) + + // Synchronize with the live sandbox but no containers. + pod := makePod(sandboxUID, "default", "live-pod") + _, err := p.Synchronize(context.Background(), []*api.PodSandbox{pod}, nil) + require.NoError(t, err) + + // The initial Reconcile(liveKeys) must have kept it. + require.DirExists(t, sandboxDir) + require.NotContains(t, p.mgr.List(), sandboxUID, "container-less sandbox is not tracked in the Manager") + + // A background reconcile tick must still preserve it (regression: it used to + // reconcile against mgr.List() only and reap the group). + p.reconcile(p.mgr) + assert.DirExists(t, sandboxDir) + + // Once the sandbox is removed, it stops being protected and is reaped. + require.NoError(t, p.RemovePodSandbox(context.Background(), pod)) + p.reconcile(p.mgr) + _, err = os.Stat(sandboxDir) + assert.True(t, os.IsNotExist(err), "sandbox mon_group should be reaped after the pod is gone") +} + +func TestReconcile_RetriesPendingRemoval(t *testing.T) { + tmpDir := t.TempDir() + p := newTestPlugin(tmpDir) + podUID := "a1b2c3d4-e5f6-7890-abcd-ef1234567890" + + // A tracked group whose earlier Remove is assumed to have failed. + _, err := p.mgr.EnsureGroup(podUID, "") + require.NoError(t, err) + require.Contains(t, p.mgr.List(), podUID) + p.markPendingRemoval(podUID) + + // The reconciler must retry Remove (not merely Reconcile, which preserves + // tracked keys) and clear the pending entry on success. + p.reconcile(p.mgr) + + assert.NotContains(t, p.mgr.List(), podUID) + assert.Empty(t, p.pendingRemovalKeys()) + _, err = os.Stat(filepath.Join(tmpDir, "mon_groups", podUID)) + assert.True(t, os.IsNotExist(err), "pending mon_group should have been removed on retry") +} + func TestPostCreateContainer_InvalidUID(t *testing.T) { p := newTestPlugin(t.TempDir()) @@ -406,9 +492,8 @@ func TestCheckRuntimeVersion(t *testing.T) { } // TestSetConfig_ReloadTearsDownTelemetry verifies that a dynamic setConfig -// reload, after telemetry has started, unregisters the OTel instruments bound -// to the old manager and starts fresh telemetry against the new manager, -// rather than leaking the old registration. +// reload, after telemetry has started, unregisters the OTel instruments and +// starts fresh telemetry, rather than leaking the old registration. func TestSetConfig_ReloadTearsDownTelemetry(t *testing.T) { groups := map[string]map[string]map[string]string{ "11111111-1111-1111-1111-111111111111": { @@ -416,7 +501,6 @@ func TestSetConfig_ReloadTearsDownTelemetry(t *testing.T) { }, } root1 := setupTestResctrl(t, groups) - root2 := setupTestResctrl(t, groups) p := newTestPlugin(root1) // Disable Prometheus so telemetry starts without binding a port. @@ -429,8 +513,8 @@ func TestSetConfig_ReloadTearsDownTelemetry(t *testing.T) { require.NotNil(t, oldReg) require.NotNil(t, oldTelem) - // Dynamic reconfiguration to a new resctrl root, telemetry still port-less. - data := []byte("resctrlPath: " + root2 + "\ntelemetry:\n prometheus:\n enabled: false\n") + // Dynamic reconfiguration (same, immutable root), telemetry still port-less. + data := []byte("resctrlPath: " + root1 + "\ntelemetry:\n prometheus:\n enabled: false\n") require.NoError(t, p.setConfig(data)) t.Cleanup(func() { if p.telemetry != nil { @@ -447,3 +531,48 @@ func TestSetConfig_ReloadTearsDownTelemetry(t *testing.T) { // The old registration was already unregistered; a second call is a no-op. assert.NoError(t, oldReg.Unregister()) } + +// TestSetConfig_RejectsRootChange verifies that changing resctrlPath on a +// running plugin (telemetry started) is rejected and the original root is +// retained. +func TestSetConfig_RejectsRootChange(t *testing.T) { + empty := map[string]map[string]map[string]string{} + root1 := setupTestResctrl(t, empty) + root2 := setupTestResctrl(t, empty) + + p := newTestPlugin(root1) + // Bring the plugin up (port-less telemetry) so the immutability guard is + // in force. + p.config.Telemetry = defaultTelemetryConfig() + p.config.Telemetry.Prometheus.Enabled = false + require.NoError(t, p.startTelemetry(context.Background())) + t.Cleanup(func() { + if p.telemetry != nil { + p.telemetry.shutdown(context.Background()) + } + }) + + err := p.setConfig([]byte("resctrlPath: " + root2 + "\ntelemetry:\n prometheus:\n enabled: false\n")) + require.Error(t, err) + assert.Contains(t, err.Error(), "cannot be changed") + assert.Equal(t, root1, p.config.ResctrlPath) +} + +// TestSetConfig_AllowsInitialRootSelection verifies that a non-default +// resctrlPath supplied before the plugin is running (no telemetry, no +// reconciler) is accepted and rebuilds the manager, rather than being rejected +// as a change to a running plugin. +func TestSetConfig_AllowsInitialRootSelection(t *testing.T) { + empty := map[string]map[string]map[string]string{} + root1 := setupTestResctrl(t, empty) + root2 := setupTestResctrl(t, empty) + + // newPlugin-equivalent initial state: config points at root1, no telemetry + // or reconciler running yet. + p := newTestPlugin(root1) + oldMgr := p.mgr + + require.NoError(t, p.setConfig([]byte("resctrlPath: "+root2+"\n"))) + assert.Equal(t, root2, p.config.ResctrlPath) + assert.NotSame(t, oldMgr, p.mgr, "manager should be rebuilt for the new root") +} diff --git a/cmd/plugins/resctrl-mon/telemetry.go b/cmd/plugins/resctrl-mon/telemetry.go index 793567e3b..1ed80371d 100644 --- a/cmd/plugins/resctrl-mon/telemetry.go +++ b/cmd/plugins/resctrl-mon/telemetry.go @@ -66,6 +66,9 @@ func defaultTelemetryConfig() telemetryConfig { var cfg telemetryConfig cfg.Prometheus.Enabled = true cfg.Prometheus.ListenAddress = ":9100" + // Match the chart/sample/README default: OTLP uses a plaintext connection + // unless the operator explicitly opts into TLS. + cfg.OTLP.Insecure = true return cfg } @@ -230,14 +233,16 @@ func (p *plugin) startTelemetry(ctx context.Context) error { if err != nil { return err } - p.telemetry = state meter := state.provider.Meter("nri-resctrl-mon") - reg, err := setupMetrics(p.mgr, cfg, meter) + reg, err := setupMetrics(p.mgr, cfg, p.config.ResctrlPath, meter) if err != nil { state.shutdown(ctx) return fmt.Errorf("metrics registration: %w", err) } + // Publish only after registration succeeds; otherwise a failed setupMetrics + // would leave p.telemetry non-nil and a later Configure would skip startup. + p.telemetry = state p.metrics = reg return nil } diff --git a/cmd/plugins/resctrl-mon/telemetry_test.go b/cmd/plugins/resctrl-mon/telemetry_test.go index 2bd308426..c856d701a 100644 --- a/cmd/plugins/resctrl-mon/telemetry_test.go +++ b/cmd/plugins/resctrl-mon/telemetry_test.go @@ -101,7 +101,7 @@ func TestTelemetryPrometheusEndpoint(t *testing.T) { _, err = mgr.EnsureGroup(podUID, "") require.NoError(t, err) - // Use a random port to avoid conflicts. + // Bind an ephemeral port to avoid conflicts on shared CI runners. cfg := defaultTelemetryConfig() cfg.Prometheus.ListenAddress = "127.0.0.1:0" cfg.PerfCounters.Enabled = false // default: suppress perf counters @@ -110,24 +110,17 @@ func TestTelemetryPrometheusEndpoint(t *testing.T) { require.NoError(t, err) defer state.shutdown(context.Background()) - // Get the actual port from the listener. - // Since we used :0, we need to start with a listener first. - // Workaround: use a fixed high port for testing. - state.shutdown(context.Background()) - - cfg.Prometheus.ListenAddress = "127.0.0.1:19100" - state, err = newTelemetry(context.Background(), cfg) - require.NoError(t, err) - defer state.shutdown(context.Background()) - meter := state.provider.Meter("nri-resctrl-mon-test") - _, err = setupMetrics(mgr, cfg, meter) + _, err = setupMetrics(mgr, cfg, root, meter) require.NoError(t, err) + // Scrape the actual address the listener bound to. + addr := state.promListener.Addr().String() + // Wait for server to be ready. time.Sleep(50 * time.Millisecond) - resp, err := http.Get("http://127.0.0.1:19100/metrics") + resp, err := http.Get("http://" + addr + "/metrics") require.NoError(t, err) defer func() { _ = resp.Body.Close() }() @@ -264,26 +257,23 @@ func TestPerfCountersGate(t *testing.T) { func TestControlGroupOf(t *testing.T) { tests := []struct { + root string path string want string }{ - {"/sys/fs/resctrl/mon_groups/abc-123", ""}, - {"/sys/fs/resctrl/COS1/mon_groups/abc-123", "COS1"}, - {"/sys/fs/resctrl/my-class/mon_groups/abc-123", "my-class"}, + {"/sys/fs/resctrl", "/sys/fs/resctrl/mon_groups/abc-123", ""}, + {"/sys/fs/resctrl", "/sys/fs/resctrl/COS1/mon_groups/abc-123", "COS1"}, + {"/sys/fs/resctrl", "/sys/fs/resctrl/my-class/mon_groups/abc-123", "my-class"}, + {"/mnt/rdt", "/mnt/rdt/mon_groups/abc-123", ""}, + {"/mnt/rdt", "/mnt/rdt/COS1/mon_groups/abc-123", "COS1"}, } for _, tt := range tests { t.Run(tt.path, func(t *testing.T) { - assert.Equal(t, tt.want, controlGroupOf(tt.path)) + assert.Equal(t, tt.want, controlGroupOf(tt.root, tt.path)) }) } } -func TestSourceFor(t *testing.T) { - assert.Equal(t, "pod", sourceFor("12345678-1234-1234-1234-123456789abc")) - assert.Equal(t, "other", sourceFor("machine-qemu-1-vm")) - assert.Equal(t, "other", sourceFor("not-a-uuid")) -} - func TestValidateTelemetryConfig(t *testing.T) { t.Run("valid defaults", func(t *testing.T) { cfg := defaultTelemetryConfig() diff --git a/deployment/helm/resctrl-mon/README.md b/deployment/helm/resctrl-mon/README.md index 212e90042..e223879d5 100644 --- a/deployment/helm/resctrl-mon/README.md +++ b/deployment/helm/resctrl-mon/README.md @@ -129,9 +129,9 @@ customize with their own values, along with the default values. | `telemetry.prometheus.enabled` | `true` | expose a `/metrics` Prometheus endpoint | | `telemetry.prometheus.listenAddress` | `":9100"` | address:port for the Prometheus HTTP listener | | `telemetry.prometheus.scrapeInterval` | `"15s"` | recommended scrape interval (set via pod annotation hint) | -| `telemetry.prometheus.namespace` | `""` | Prometheus metric prefix (empty = `resctrl_`) | +| `telemetry.prometheus.namespace` | `""` | Prometheus metric prefix (empty = no prefix, e.g. `l3_*`/`perf_*`) | | `telemetry.otlp.enabled` | `false` | push metrics via OTLP | -| `telemetry.otlp.endpoint` | `""` | OTLP receiver endpoint (e.g. `otel-collector:4317`) | +| `telemetry.otlp.endpoint` | `""` | OTLP receiver endpoint (e.g. `otel-collector-resctrl:4317`) | | `telemetry.otlp.protocol` | `grpc` | `grpc` or `http` | | `telemetry.otlp.interval` | `15s` | OTLP export interval | | `telemetry.otlp.insecure` | `true` | disable TLS for OTLP connection | @@ -156,7 +156,7 @@ in your Prometheus configuration with the desired `scrape_interval`. | Component | Minimum Version | Notes | | ---------------- | --------------- | --------------------------------------------------------------------- | -| Linux kernel | 7.0+ | Required for AET (`rdt=perf` Kconfig). CMT/MBM works on 5.x+. | +| Linux kernel | 5.x+ | CMT/MBM on 5.x+; AET perf/energy counters need `rdt=perf` (pending upstream). | | containerd | 1.7.0+ | NRI support required. | | CRI-O | 1.36.0+ | Provides container PIDs via NRI `LinuxContainer.Pid`. | | Kubernetes | 1.24+ | DaemonSet and NRI socket conventions. | @@ -167,7 +167,11 @@ in your Prometheus configuration with the desired `scrape_interval`. | Counter family | Kernel Kconfig | Available since | | ------------------------------- | ----------------------------- | --------------- | | `llc_occupancy`, `mbm_*` | `CONFIG_X86_CPU_RESCTRL` | 5.x | -| `c1_res`, `stalls_*`, `energy_*` | `CONFIG_X86_CPU_RESCTRL` + `rdt=perf` boot param | 7.0 (under review) | +| `c1_res`, `stalls_*`, `energy_*` | `CONFIG_X86_CPU_RESCTRL` + `rdt=perf` boot param | pending (under review) | + +> **Note:** `rdt=perf` kernel support is still under review upstream and is not +> yet part of a released kernel. The "Available since" version for these +> counters is TBD and will be recorded once the change lands. ## Optional: OTel Collector sidecar diff --git a/deployment/helm/resctrl-mon/optional/grafana-resctrl-perf-counters.json b/deployment/helm/resctrl-mon/optional/grafana-resctrl-perf-counters.json index d07f05714..d3cd05be7 100644 --- a/deployment/helm/resctrl-mon/optional/grafana-resctrl-perf-counters.json +++ b/deployment/helm/resctrl-mon/optional/grafana-resctrl-perf-counters.json @@ -8,6 +8,20 @@ "links": [], "templating": { "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Prometheus", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, { "current": { "selected": true, @@ -20,7 +34,7 @@ }, "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "definition": "label_values(kube_pod_info, pod)", "hide": 0, @@ -49,7 +63,7 @@ }, "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "definition": "label_values(kube_pod_info, namespace)", "hide": 0, @@ -84,7 +98,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "Total power across all monitored pods", "fieldConfig": { @@ -142,12 +156,12 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum(rate({__name__=\"perf.core.energy_joules_total\", \"resctrl.group.source\"=\"pod\"}[$__rate_interval]))", + "expr": "sum(rate({__name__=\"perf_core_energy_joules_total\", resctrl_group_source=\"pod\"}[$__rate_interval]))", "interval": "5s", - "legendFormat": "{{k8s.pod.uid}}", + "legendFormat": "{{k8s_pod_uid}}", "refId": "A" } ], @@ -157,7 +171,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "Total CPU activity rate across all monitored pods", "fieldConfig": { @@ -207,12 +221,12 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum(rate({__name__=\"perf.activity_farads_total\", \"resctrl.group.source\"=\"pod\"}[$__rate_interval]))", + "expr": "sum(rate({__name__=\"perf_activity_farads_total\", resctrl_group_source=\"pod\"}[$__rate_interval]))", "interval": "5s", - "legendFormat": "{{k8s.pod.uid}}", + "legendFormat": "{{k8s_pod_uid}}", "refId": "A" } ], @@ -222,7 +236,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "Total unhalted core cycles/sec across all monitored pods", "fieldConfig": { @@ -272,12 +286,12 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum(rate({__name__=\"perf.unhalted.core.cycles_total\", \"resctrl.group.source\"=\"pod\"}[$__rate_interval]))", + "expr": "sum(rate({__name__=\"perf_unhalted_core_cycles_total\", resctrl_group_source=\"pod\"}[$__rate_interval]))", "interval": "5s", - "legendFormat": "{{k8s.pod.uid}}", + "legendFormat": "{{k8s_pod_uid}}", "refId": "A" } ], @@ -287,7 +301,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "Total retired micro-ops/sec across all monitored pods", "fieldConfig": { @@ -337,12 +351,12 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum(rate({__name__=\"perf.uops.retired_total\", \"resctrl.group.source\"=\"pod\"}[$__rate_interval]))", + "expr": "sum(rate({__name__=\"perf_uops_retired_total\", resctrl_group_source=\"pod\"}[$__rate_interval]))", "interval": "5s", - "legendFormat": "{{k8s.pod.uid}}", + "legendFormat": "{{k8s_pod_uid}}", "refId": "A" } ], @@ -364,7 +378,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "Per-pod power consumption derived from Intel AET energy counter", "fieldConfig": { @@ -416,10 +430,10 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum by (pod) (\n rate({__name__=\"perf.core.energy_joules_total\"}[$__rate_interval])\n * on(\"k8s.pod.uid\") group_left(pod, namespace)\nlabel_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\")\n)", + "expr": "sum by (pod) (\n rate({__name__=\"perf_core_energy_joules_total\"}[$__rate_interval])\n * on(k8s_pod_uid) group_left(pod, namespace)\nlabel_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\")\n)", "interval": "5s", "legendFormat": "{{pod}}", "refId": "A" @@ -443,7 +457,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "Per-pod CPU activity rate (capacitance proxy from AET)", "fieldConfig": { @@ -495,10 +509,10 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum by (pod) (\n rate({__name__=\"perf.activity_farads_total\"}[$__rate_interval])\n * on(\"k8s.pod.uid\") group_left(pod, namespace)\nlabel_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\")\n)", + "expr": "sum by (pod) (\n rate({__name__=\"perf_activity_farads_total\"}[$__rate_interval])\n * on(k8s_pod_uid) group_left(pod, namespace)\nlabel_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\")\n)", "interval": "5s", "legendFormat": "{{pod}}", "refId": "A" @@ -522,7 +536,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "Ratio of unhalted core cycles to reference cycles per pod. 1.0 = base frequency, >1.0 = turbo, <1.0 = throttled.", "fieldConfig": { @@ -592,10 +606,10 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum by (pod) (\n (\n sum by (\"k8s.pod.uid\") (rate({__name__=\"perf.unhalted.core.cycles_total\"}[$__rate_interval]))\n / clamp_min(sum by (\"k8s.pod.uid\") (rate({__name__=\"perf.unhalted.ref.cycles_total\"}[$__rate_interval])), 1)\n )\n * on(\"k8s.pod.uid\") group_left(pod, namespace)\n label_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\")\n)", + "expr": "sum by (pod) (\n (\n sum by (k8s_pod_uid) (rate({__name__=\"perf_unhalted_core_cycles_total\"}[$__rate_interval]))\n / clamp_min(sum by (k8s_pod_uid) (rate({__name__=\"perf_unhalted_ref_cycles_total\"}[$__rate_interval])), 1)\n )\n * on(k8s_pod_uid) group_left(pod, namespace)\n label_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\")\n)", "interval": "5s", "legendFormat": "{{pod}}", "refId": "A" @@ -619,7 +633,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "Per-pod retired micro-operations per second", "fieldConfig": { @@ -671,10 +685,10 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum by (pod) (\n rate({__name__=\"perf.uops.retired_total\"}[$__rate_interval])\n * on(\"k8s.pod.uid\") group_left(pod, namespace)\nlabel_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\")\n)", + "expr": "sum by (pod) (\n rate({__name__=\"perf_uops_retired_total\"}[$__rate_interval])\n * on(k8s_pod_uid) group_left(pod, namespace)\nlabel_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\")\n)", "interval": "5s", "legendFormat": "{{pod}}", "refId": "A" @@ -698,7 +712,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "Per-pod C1 residency rate", "fieldConfig": { @@ -750,10 +764,10 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum by (pod) (\n rate({__name__=\"perf.c1.res_total\"}[$__rate_interval])\n * on(\"k8s.pod.uid\") group_left(pod, namespace)\nlabel_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\")\n)", + "expr": "sum by (pod) (\n rate({__name__=\"perf_c1_res_total\"}[$__rate_interval])\n * on(k8s_pod_uid) group_left(pod, namespace)\nlabel_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\")\n)", "interval": "5s", "legendFormat": "{{pod}}", "refId": "A" @@ -765,7 +779,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "Per-pod C6 residency rate", "fieldConfig": { @@ -817,10 +831,10 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum by (pod) (\n rate({__name__=\"perf.c6.res_total\"}[$__rate_interval])\n * on(\"k8s.pod.uid\") group_left(pod, namespace)\nlabel_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\")\n)", + "expr": "sum by (pod) (\n rate({__name__=\"perf_c6_res_total\"}[$__rate_interval])\n * on(k8s_pod_uid) group_left(pod, namespace)\nlabel_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\")\n)", "interval": "5s", "legendFormat": "{{pod}}", "refId": "A" @@ -844,7 +858,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "Per-pod LLC hit stalls per second", "fieldConfig": { @@ -896,10 +910,10 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum by (pod) (\n rate({__name__=\"perf.stalls.llc.hit_total\"}[$__rate_interval])\n * on(\"k8s.pod.uid\") group_left(pod, namespace)\nlabel_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\")\n)", + "expr": "sum by (pod) (\n rate({__name__=\"perf_stalls_llc_hit_total\"}[$__rate_interval])\n * on(k8s_pod_uid) group_left(pod, namespace)\nlabel_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\")\n)", "interval": "5s", "legendFormat": "{{pod}}", "refId": "A" @@ -911,7 +925,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "Per-pod LLC miss stalls per second", "fieldConfig": { @@ -963,10 +977,10 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum by (pod) (\n rate({__name__=\"perf.stalls.llc.miss_total\"}[$__rate_interval])\n * on(\"k8s.pod.uid\") group_left(pod, namespace)\nlabel_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\")\n)", + "expr": "sum by (pod) (\n rate({__name__=\"perf_stalls_llc_miss_total\"}[$__rate_interval])\n * on(k8s_pod_uid) group_left(pod, namespace)\nlabel_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{pod=~\"$pod_name\", namespace=~\"$namespace\"}[$__range])), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\")\n)", "interval": "5s", "legendFormat": "{{pod}}", "refId": "A" diff --git a/deployment/helm/resctrl-mon/optional/grafana-resctrl-pod-energy.json b/deployment/helm/resctrl-mon/optional/grafana-resctrl-pod-energy.json index 15d2bff0c..5af2fdf9d 100644 --- a/deployment/helm/resctrl-mon/optional/grafana-resctrl-pod-energy.json +++ b/deployment/helm/resctrl-mon/optional/grafana-resctrl-pod-energy.json @@ -22,7 +22,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "Sum of power consumed by all monitored pods (from Intel AET via resctrl)", "fieldConfig": { @@ -82,9 +82,9 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate({__name__=\"perf.core.energy_joules_total\", \"resctrl.group.source\"=\"pod\"}[$__rate_interval]))", + "expr": "sum(rate({__name__=\"perf_core_energy_joules_total\", resctrl_group_source=\"pod\"}[$__rate_interval]))", "legendFormat": "Total Pod Power", "refId": "A", "interval": "5s" @@ -96,7 +96,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "Energy consumed by all monitored pods in the displayed time window (Wh)", "fieldConfig": { @@ -144,9 +144,9 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(increase({__name__=\"perf.core.energy_joules_total\", \"resctrl.group.source\"=\"pod\"}[$__range])) / 3600", + "expr": "sum(increase({__name__=\"perf_core_energy_joules_total\", resctrl_group_source=\"pod\"}[$__range])) / 3600", "legendFormat": "Total Energy", "refId": "A", "interval": "5s" @@ -158,7 +158,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "Percentage of total activity attributable to monitored pods (AET activity counters, measured in Farads)", "fieldConfig": { @@ -214,15 +214,15 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate({__name__=\"perf.activity_farads_total\", \"resctrl.group.source\"=\"pod\"}[$__rate_interval])) / sum(rate({__name__=\"perf.activity_farads_total\"}[$__rate_interval])) * 100", - "legendFormat": "Pod Activity Share", + "expr": "sum(rate({__name__=\"perf_activity_farads_total\", resctrl_group_source=\"pod\"}[$__rate_interval])) / sum(rate({__name__=\"perf_activity_farads_total\"}[$__rate_interval])) * 100", + "legendFormat": "Pod-Attributed", "refId": "A", "interval": "5s" } ], - "title": "Pod Activity Share", + "title": "Pod-Attributed Activity (share of observed)", "type": "stat" }, { @@ -240,7 +240,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "Power consumption per pod from Intel AET core_energy counters", "fieldConfig": { @@ -321,10 +321,10 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum by (pod) (rate({__name__=\"perf.core.energy_joules_total\", \"resctrl.group.source\"=\"pod\"}[$__rate_interval]) * on(\"k8s.pod.uid\") group_left(pod, namespace) label_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info[$__range])), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\"))", - "legendFormat": "{{pod_name}}", + "expr": "sum by (pod) (rate({__name__=\"perf_core_energy_joules_total\", resctrl_group_source=\"pod\"}[$__rate_interval]) * on(k8s_pod_uid) group_left(pod, namespace) label_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{namespace=~\"$namespace\"}[$__range])), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\"))", + "legendFormat": "{{pod}}", "refId": "A", "interval": "5s" } @@ -335,7 +335,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "Total energy (Joules) consumed by each workload over the selected range (donut chart). Pods sharing a common name are summed; pods that started and stopped within the range are included.", "fieldConfig": { @@ -393,9 +393,9 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum by (workload) (increase({__name__=\"perf.core.energy_joules_total\", \"resctrl.group.source\"=\"pod\"}[$__range]) * on(\"k8s.pod.uid\") group_left(workload) label_replace(label_replace(label_replace(label_replace(max by (uid, pod, namespace, created_by_name) (last_over_time(kube_pod_info[$__range])), \"workload\", \"$1\", \"pod\", \"(.+)\"), \"workload\", \"$1\", \"created_by_name\", \"(.+)\"), \"workload\", \"$1\", \"workload\", \"(.+?)-[bcdfghjklmnpqrstvwxz2-9]{6,10}$\"), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\"))", + "expr": "sum by (workload) (increase({__name__=\"perf_core_energy_joules_total\", resctrl_group_source=\"pod\"}[$__range]) * on(k8s_pod_uid) group_left(workload) label_replace(label_replace(label_replace(label_replace(max by (uid, pod, namespace, created_by_name) (last_over_time(kube_pod_info{namespace=~\"$namespace\"}[$__range])), \"workload\", \"$1\", \"pod\", \"(.+)\"), \"workload\", \"$1\", \"created_by_name\", \"(.+)\"), \"workload\", \"$1\", \"workload\", \"(.+?)-[bcdfghjklmnpqrstvwxz2-9]{6,10}$\"), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\"))", "legendFormat": "{{workload}}", "refId": "A" } @@ -418,7 +418,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "Activity rate (F/s) per pod from AET activity counters. Activity is measured in Farads and represents frequency-independent work done.", "fieldConfig": { @@ -499,10 +499,10 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum by (pod) (rate({__name__=\"perf.activity_farads_total\", \"resctrl.group.source\"=\"pod\"}[$__rate_interval]) * on(\"k8s.pod.uid\") group_left(pod, namespace) label_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info[$__range])), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\"))", - "legendFormat": "{{pod_name}}", + "expr": "sum by (pod) (rate({__name__=\"perf_activity_farads_total\", resctrl_group_source=\"pod\"}[$__rate_interval]) * on(k8s_pod_uid) group_left(pod, namespace) label_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{namespace=~\"$namespace\"}[$__range])), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\"))", + "legendFormat": "{{pod}}", "refId": "A", "interval": "5s" } @@ -513,7 +513,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "Total activity (Farads) accumulated by each workload over the selected range (donut chart). Activity is frequency-independent work. Pods sharing a common name are summed; pods that started and stopped within the range are included.", "fieldConfig": { @@ -571,9 +571,9 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum by (workload) (increase({__name__=\"perf.activity_farads_total\", \"resctrl.group.source\"=\"pod\"}[$__range]) * on(\"k8s.pod.uid\") group_left(workload) label_replace(label_replace(label_replace(label_replace(max by (uid, pod, namespace, created_by_name) (last_over_time(kube_pod_info[$__range])), \"workload\", \"$1\", \"pod\", \"(.+)\"), \"workload\", \"$1\", \"created_by_name\", \"(.+)\"), \"workload\", \"$1\", \"workload\", \"(.+?)-[bcdfghjklmnpqrstvwxz2-9]{6,10}$\"), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\"))", + "expr": "sum by (workload) (increase({__name__=\"perf_activity_farads_total\", resctrl_group_source=\"pod\"}[$__range]) * on(k8s_pod_uid) group_left(workload) label_replace(label_replace(label_replace(label_replace(max by (uid, pod, namespace, created_by_name) (last_over_time(kube_pod_info{namespace=~\"$namespace\"}[$__range])), \"workload\", \"$1\", \"pod\", \"(.+)\"), \"workload\", \"$1\", \"created_by_name\", \"(.+)\"), \"workload\", \"$1\", \"workload\", \"(.+?)-[bcdfghjklmnpqrstvwxz2-9]{6,10}$\"), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\"))", "legendFormat": "{{workload}}", "refId": "A" } @@ -596,7 +596,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "All monitored pods with their current power draw, activity rate, and cumulative energy", "fieldConfig": { @@ -774,9 +774,9 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum by (pod, namespace) (rate({__name__=\"perf.core.energy_joules_total\", \"resctrl.group.source\"=\"pod\"}[$__rate_interval]) * on(\"k8s.pod.uid\") group_left(pod, namespace) label_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info[$__range])), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\"))", + "expr": "sum by (pod, namespace) (rate({__name__=\"perf_core_energy_joules_total\", resctrl_group_source=\"pod\"}[$__rate_interval]) * on(k8s_pod_uid) group_left(pod, namespace) label_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{namespace=~\"$namespace\"}[$__range])), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\"))", "format": "table", "instant": true, "legendFormat": "", @@ -786,9 +786,9 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum by (pod, namespace) (rate({__name__=\"perf.activity_farads_total\", \"resctrl.group.source\"=\"pod\"}[$__rate_interval]) * on(\"k8s.pod.uid\") group_left(pod, namespace) label_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info[$__range])), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\"))", + "expr": "sum by (pod, namespace) (rate({__name__=\"perf_activity_farads_total\", resctrl_group_source=\"pod\"}[$__rate_interval]) * on(k8s_pod_uid) group_left(pod, namespace) label_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{namespace=~\"$namespace\"}[$__range])), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\"))", "format": "table", "instant": true, "legendFormat": "", @@ -798,9 +798,9 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum by (pod, namespace) (increase({__name__=\"perf.core.energy_joules_total\", \"resctrl.group.source\"=\"pod\"}[$__range]) * on(\"k8s.pod.uid\") group_left(pod, namespace) label_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info[$__range])), \"k8s.pod.uid\", \"$1\", \"uid\", \"(.+)\")) / 1000", + "expr": "sum by (pod, namespace) (increase({__name__=\"perf_core_energy_joules_total\", resctrl_group_source=\"pod\"}[$__range]) * on(k8s_pod_uid) group_left(pod, namespace) label_replace(max by (uid, pod, namespace) (last_over_time(kube_pod_info{namespace=~\"$namespace\"}[$__range])), \"k8s_pod_uid\", \"$1\", \"uid\", \"(.+)\")) / 1000", "format": "table", "instant": true, "legendFormat": "", @@ -846,9 +846,9 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, - "description": "Total power per CPU package (domain) including unmonitored background workloads", + "description": "Total power per CPU package (domain) across monitored pods (root/system counters are not observed)", "fieldConfig": { "defaults": { "color": { @@ -924,23 +924,23 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum by (\"domain.id\") (rate({__name__=\"perf.core.energy_joules_total\"}[$__rate_interval]))", - "legendFormat": "{{domain}}", + "expr": "sum by (domain_id) (rate({__name__=\"perf_core_energy_joules_total\", resctrl_group_source=\"pod\"}[$__rate_interval]))", + "legendFormat": "{{domain_id}}", "refId": "A", "interval": "5s" } ], - "title": "Package Total Power (all workloads)", + "title": "Package Power (monitored pods)", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, - "description": "Total activity per package (domain) in F/s \u2014 includes all workloads on the system", + "description": "Total activity per package (domain) in F/s \u2014 across monitored pods (root/system counters are not observed)", "fieldConfig": { "defaults": { "color": { @@ -1016,15 +1016,15 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, - "expr": "sum by (\"domain.id\") (rate({__name__=\"perf.activity_farads_total\"}[$__rate_interval]))", - "legendFormat": "{{domain}}", + "expr": "sum by (domain_id) (rate({__name__=\"perf_activity_farads_total\", resctrl_group_source=\"pod\"}[$__rate_interval]))", + "legendFormat": "{{domain_id}}", "refId": "A", "interval": "5s" } ], - "title": "Package Total Activity", + "title": "Package Activity (monitored pods)", "type": "timeseries" } ], @@ -1057,7 +1057,7 @@ "current": {}, "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "definition": "label_values(kube_pod_info, namespace)", "hide": 0, diff --git a/deployment/helm/resctrl-mon/optional/otel-collector-agent.yaml b/deployment/helm/resctrl-mon/optional/otel-collector-agent.yaml index 6a70b4c15..f9331e68c 100644 --- a/deployment/helm/resctrl-mon/optional/otel-collector-agent.yaml +++ b/deployment/helm/resctrl-mon/optional/otel-collector-agent.yaml @@ -24,6 +24,13 @@ data: grpc: endpoint: 0.0.0.0:4317 processors: + # resctrl-mon emits k8s.pod.uid as a data-point attribute. The + # k8sattributes pod_association can only match on resource attributes, + # so promote it (and any other shared data-point attributes) to the + # resource level first with groupbyattrs. + groupbyattrs: + keys: + - k8s.pod.uid k8sattributes: auth_type: serviceAccount extract: @@ -46,7 +53,7 @@ data: pipelines: metrics: receivers: [otlp] - processors: [k8sattributes, batch] + processors: [groupbyattrs, k8sattributes, batch] exporters: [prometheus] --- apiVersion: apps/v1 @@ -90,3 +97,30 @@ spec: - name: config configMap: name: otel-collector-resctrl-config +--- +# Service exposing the OTLP gRPC receiver so the plugin can reach the +# collector at otel-collector-resctrl.monitoring.svc:4317. Because the +# collector runs as a DaemonSet, this ClusterIP Service load-balances across +# all node-local collectors; enrichment is correct regardless of which pod +# handles a request (k8sattributes has cluster-wide pod visibility). For +# strict node-local delivery, have the plugin target the node's host IP +# (status.hostIP) instead. +apiVersion: v1 +kind: Service +metadata: + name: otel-collector-resctrl + namespace: monitoring + labels: + app.kubernetes.io/name: otel-collector-resctrl +spec: + selector: + app.kubernetes.io/name: otel-collector-resctrl + ports: + - name: otlp-grpc + port: 4317 + targetPort: otlp-grpc + protocol: TCP + - name: prom-export + port: 8889 + targetPort: prom-export + protocol: TCP diff --git a/deployment/helm/resctrl-mon/optional/otel-collector-rbac.yaml b/deployment/helm/resctrl-mon/optional/otel-collector-rbac.yaml index 0114a49ad..672693451 100644 --- a/deployment/helm/resctrl-mon/optional/otel-collector-rbac.yaml +++ b/deployment/helm/resctrl-mon/optional/otel-collector-rbac.yaml @@ -1,6 +1,14 @@ # RBAC for the OTel Collector k8sattributes processor. # Grants read access to pod metadata so the processor can enrich metrics. --- +# The reference manifests deploy into the monitoring namespace. Create it here +# so a clean cluster can `kubectl apply` these files without a prior step; +# apply is idempotent if the namespace already exists. +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring +--- apiVersion: v1 kind: ServiceAccount metadata: diff --git a/deployment/helm/resctrl-mon/values.yaml b/deployment/helm/resctrl-mon/values.yaml index e38c43d20..68bc01cbe 100644 --- a/deployment/helm/resctrl-mon/values.yaml +++ b/deployment/helm/resctrl-mon/values.yaml @@ -10,10 +10,10 @@ telemetry: enabled: true listenAddress: ":9100" scrapeInterval: "15s" # recommended scrape interval (annotation hint) - namespace: "" # empty = resctrl_* Prometheus metric names + namespace: "" # empty = no prefix (l3_*/perf_* metric names) otlp: enabled: false - endpoint: "" # e.g. "otel-collector.monitoring.svc:4317" + endpoint: "" # e.g. "otel-collector-resctrl.monitoring.svc:4317" protocol: grpc # grpc | http interval: 15s insecure: true