From 9f54f5a137ad901f2b9c43c67f8e4355ed641bd3 Mon Sep 17 00:00:00 2001 From: mayankpande88 Date: Thu, 20 Aug 2026 16:48:21 +0530 Subject: [PATCH 1/3] fix(nb-36647): drop the image from Job-collapsed backoff fingerprints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #580, caught verifying that change on the dev cluster. #580 collapsed a Job owner to its family so repeated runs would share a fingerprint, but kept the failing image in the hash. For a one-Job-per- image creator the image IS the per-run identity, so it re-forked the fingerprint immediately: after #580 shipped, every trivy-image-scan Job in nudgebee-agent-dev still had its own fingerprint, each pulling a different image under scan. Drop the image only when we collapsed a Job owner. Ordinary workloads keep it, so a single typo'd container is still its own Finding. The failing image remains in the evidence blocks either way. Measured on 30d of prod: job-owned 3961 -> 9 fingerprints. Every other owner kind is unchanged (deployment 35, daemonset 8, statefulset 3) — the image component only ever forked identity for Job-owned Pods. --- runner/pkg/triggers/predicates.go | 29 +++++++++------ runner/pkg/triggers/predicates_test.go | 50 ++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 11 deletions(-) diff --git a/runner/pkg/triggers/predicates.go b/runner/pkg/triggers/predicates.go index e693203..a902820 100644 --- a/runner/pkg/triggers/predicates.go +++ b/runner/pkg/triggers/predicates.go @@ -290,8 +290,9 @@ func oomKilledEnrichBlocks(obj, _ map[string]any, ec EnrichContext) []EvidenceBl // imagePullBackoffMatcher fires when any container is currently in // ImagePullBackOff or ErrImagePull. Same kubewatch pointer-aliasing // caveat as pod_crash_loop — we can't rely on oldObj→obj transition -// detection. The fingerprint (`owner, image`) dedupes: distinct bad -// images each fire once per 10-min window. +// detection. The fingerprint is (`owner, image`) for ordinary workloads, +// and (`job family`) for Job-owned Pods — see FingerprintFn for why the +// image is deliberately excluded in the Job case. func imagePullBackoffMatcher() MatcherSpec { return MatcherSpec{ Name: "image_pull_backoff", @@ -306,24 +307,30 @@ func imagePullBackoffMatcher() MatcherSpec { }, FingerprintFn: func(obj map[string]any) string { ns, name := metaNS(obj), metaName(obj) + // The bad image discriminates between containers of one workload: + // an operator typo'd a single container and the rest are fine, so + // those should be distinct Findings. + image := firstFailingImage(obj) owner := ResolveOwner(obj) if owner.Name != "" { name = owner.Name // A Pod owned by a Job resolves to that Job, whose name // carries a per-run suffix — so a creator that runs one Job - // per unit of work (our image scanner: one Job per image) - // produced a brand-new fingerprint every run and never - // chained. Collapse to the job family (#36647). + // per unit of work produced a brand-new fingerprint every run + // and never chained. Collapse to the job family (#36647). if owner.Kind == "job" { name = JobFamily(name) + // ...and drop the image. For a one-Job-per-image creator + // (our image scanner) the image IS the per-run identity, + // so keeping it re-introduces exactly the fanout the job + // family collapse just removed — verified on the dev + // cluster after #580 shipped: every scan Job still had its + // own fingerprint because each pulls a different image. + // The failing image stays in the evidence blocks either + // way; it just no longer forks the identity. + image = "" } } - // Include the bad image — different bad images on the same - // workload should be distinct Findings (operator just typo'd - // one container, the rest are fine). This also keeps per-image - // resolution after the job-family collapse above: one scan Job - // family with two bad images is still two Findings. - image := firstFailingImage(obj) return fp("image_pull_backoff_reporter", ns, name, image) }, EnrichBlocks: imagePullBackoffEnrichBlocks, diff --git a/runner/pkg/triggers/predicates_test.go b/runner/pkg/triggers/predicates_test.go index 377e0dc..3b00055 100644 --- a/runner/pkg/triggers/predicates_test.go +++ b/runner/pkg/triggers/predicates_test.go @@ -771,6 +771,56 @@ func TestImagePullBackoff_JobOwnedPodsCollapseToJobFamily(t *testing.T) { } } +func TestImagePullBackoff_JobFamilyCollapsesAcrossDifferentImages(t *testing.T) { + // Regression for what #580 missed: a one-Job-per-image creator gives every + // run a different image, so keeping the image in the fingerprint re-forked + // the identity even after the job-family collapse. Observed on the dev + // cluster — every trivy-image-scan Job still had its own fingerprint. + mk := func(job, image string) map[string]any { + return asObj(t, `{ + "metadata":{ + "name":"`+job+`-p9x2","namespace":"scan", + "ownerReferences":[{"kind":"Job","name":"`+job+`","controller":true}] + }, + "status":{"containerStatuses":[ + {"name":"scan","image":"`+image+`", + "state":{"waiting":{"reason":"ErrImagePull"}}} + ]} + }`) + } + m := imagePullBackoffMatcher() + a := m.FingerprintFn(mk("trivy-image-scan-020c9475", "reg/nudgebee-ticket-server:2026-08-20T09-31-55_2a48d6b")) + b := m.FingerprintFn(mk("trivy-image-scan-9d22cbf9", "reg/nudgebee-llm-server:2026-08-20T09-12-54_b8f84d9")) + if a != b { + t.Error("scan Jobs of one family must share a fingerprint even when each pulls a different image") + } + // A different family in the same namespace stays distinct. + if c := m.FingerprintFn(mk("kube-bench-scan-177dde3a", "reg/kube-bench:v1")); c == a { + t.Error("a different job family must not collapse into the same fingerprint") + } +} + +func TestImagePullBackoff_NonJobOwnersKeepImageDiscrimination(t *testing.T) { + // The image must still split Findings for ordinary workloads — one typo'd + // container should not hide behind a sibling's good image. + mk := func(image string) map[string]any { + return asObj(t, `{ + "metadata":{ + "name":"api-7f9d8c5b6d-aaaaa","namespace":"prod", + "ownerReferences":[{"kind":"ReplicaSet","name":"api-7f9d8c5b6d","controller":true}] + }, + "status":{"containerStatuses":[ + {"name":"app","image":"`+image+`", + "state":{"waiting":{"reason":"ImagePullBackOff"}}} + ]} + }`) + } + m := imagePullBackoffMatcher() + if m.FingerprintFn(mk("registry/api:bad-v1")) == m.FingerprintFn(mk("registry/api:bad-v2")) { + t.Error("different bad images on a Deployment must stay distinct") + } +} + func TestImagePullBackoff_DeploymentReplicasStillCollapse(t *testing.T) { // Guards the behaviour that already worked, so the job-family change // above cannot regress it. From da3f60f8353d95c8e75c6fbfbc7b40717bfd2e53 Mon Sep 17 00:00:00 2001 From: mayankpande88 Date: Thu, 20 Aug 2026 16:54:18 +0530 Subject: [PATCH 2/3] refactor(triggers): skip firstFailingImage when the owner is a Job Review feedback on #582: the image was computed and then discarded for every Job-owned Pod, walking all container statuses for a result the fingerprint never uses. Hoist the job-owned check and compute the image only when it is part of the identity. No behaviour change. --- runner/pkg/triggers/predicates.go | 41 +++++++++++++++++-------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/runner/pkg/triggers/predicates.go b/runner/pkg/triggers/predicates.go index a902820..3e294f3 100644 --- a/runner/pkg/triggers/predicates.go +++ b/runner/pkg/triggers/predicates.go @@ -307,30 +307,35 @@ func imagePullBackoffMatcher() MatcherSpec { }, FingerprintFn: func(obj map[string]any) string { ns, name := metaNS(obj), metaName(obj) - // The bad image discriminates between containers of one workload: - // an operator typo'd a single container and the rest are fine, so - // those should be distinct Findings. - image := firstFailingImage(obj) owner := ResolveOwner(obj) + // A Pod owned by a Job resolves to that Job, whose name carries a + // per-run suffix — so a creator that runs one Job per unit of work + // produced a brand-new fingerprint every run and never chained. + // Collapse to the job family (#36647). + jobOwned := owner.Name != "" && owner.Kind == "job" if owner.Name != "" { name = owner.Name - // A Pod owned by a Job resolves to that Job, whose name - // carries a per-run suffix — so a creator that runs one Job - // per unit of work produced a brand-new fingerprint every run - // and never chained. Collapse to the job family (#36647). - if owner.Kind == "job" { + if jobOwned { name = JobFamily(name) - // ...and drop the image. For a one-Job-per-image creator - // (our image scanner) the image IS the per-run identity, - // so keeping it re-introduces exactly the fanout the job - // family collapse just removed — verified on the dev - // cluster after #580 shipped: every scan Job still had its - // own fingerprint because each pulls a different image. - // The failing image stays in the evidence blocks either - // way; it just no longer forks the identity. - image = "" } } + // The failing image discriminates between containers of one + // workload: an operator typo'd a single container and the rest are + // fine, so those stay distinct Findings. + // + // It is deliberately excluded for Job-owned Pods. For a + // one-Job-per-image creator (our image scanner) the image IS the + // per-run identity, so keeping it re-introduces exactly the fanout + // the job-family collapse just removed — verified on the dev + // cluster after #580 shipped: every scan Job still had its own + // fingerprint because each pulls a different image. The image stays + // in the evidence blocks either way; it just no longer forks the + // identity. Skipping the call also avoids walking every container + // status for a result we would discard. + var image string + if !jobOwned { + image = firstFailingImage(obj) + } return fp("image_pull_backoff_reporter", ns, name, image) }, EnrichBlocks: imagePullBackoffEnrichBlocks, From 9ff5438b903c5d7d44dc8e600634d759be0e7094 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 20 Aug 2026 14:12:13 +0000 Subject: [PATCH 3/3] chore: update image tags for main release --- charts/nudgebee-agent/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/nudgebee-agent/values.yaml b/charts/nudgebee-agent/values.yaml index 196e02d..e361202 100644 --- a/charts/nudgebee-agent/values.yaml +++ b/charts/nudgebee-agent/values.yaml @@ -65,7 +65,7 @@ runnerServiceAccount: runner: image: repository: ghcr.io/nudgebee/nudgebee-agent - tag: 2026-08-20T09-53-59_1e7426c7f4b8555ec2c620ac834c256ea1aceac7 + tag: 2026-08-20T12-43-13_7a2f3c93f2575994b793bb376c5b7e1f5c60d7ed # Image template the pod_profiler action launches debugger pods from. # The agent substitutes `{}` for the variant (bpf, jvm, python, perf, ruby). # Surfaces as PROFILER_IMAGE; leave empty to fall back to the binary default.