Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion charts/nudgebee-agent/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ runnerServiceAccount:
runner:
image:
repository: ghcr.io/nudgebee/nudgebee-agent
tag: 2026-08-20T09-53-59_1e7426c7f4b8555ec2c620ac834c256ea1aceac7
tag: 2026-08-20T12-43-13_7a2f3c93f2575994b793bb376c5b7e1f5c60d7ed
# Image template the pod_profiler action launches debugger pods from.
# The agent substitutes `{}` for the variant (bpf, jvm, python, perf, ruby).
# Surfaces as PROFILER_IMAGE; leave empty to fall back to the binary default.
Expand Down
40 changes: 26 additions & 14 deletions runner/pkg/triggers/predicates.go
Original file line number Diff line number Diff line change
Expand Up @@ -290,8 +290,9 @@ func oomKilledEnrichBlocks(obj, _ map[string]any, ec EnrichContext) []EvidenceBl
// imagePullBackoffMatcher fires when any container is currently in
// ImagePullBackOff or ErrImagePull. Same kubewatch pointer-aliasing
// caveat as pod_crash_loop — we can't rely on oldObj→obj transition
// detection. The fingerprint (`owner, image`) dedupes: distinct bad
// images each fire once per 10-min window.
// detection. The fingerprint is (`owner, image`) for ordinary workloads,
// and (`job family`) for Job-owned Pods — see FingerprintFn for why the
// image is deliberately excluded in the Job case.
func imagePullBackoffMatcher() MatcherSpec {
return MatcherSpec{
Name: "image_pull_backoff",
Expand All @@ -307,23 +308,34 @@ func imagePullBackoffMatcher() MatcherSpec {
FingerprintFn: func(obj map[string]any) string {
ns, name := metaNS(obj), metaName(obj)
owner := ResolveOwner(obj)
// A Pod owned by a Job resolves to that Job, whose name carries a
// per-run suffix — so a creator that runs one Job per unit of work
// produced a brand-new fingerprint every run and never chained.
// Collapse to the job family (#36647).
jobOwned := owner.Name != "" && owner.Kind == "job"
if owner.Name != "" {
name = owner.Name
// A Pod owned by a Job resolves to that Job, whose name
// carries a per-run suffix — so a creator that runs one Job
// per unit of work (our image scanner: one Job per image)
// produced a brand-new fingerprint every run and never
// chained. Collapse to the job family (#36647).
if owner.Kind == "job" {
if jobOwned {
name = JobFamily(name)
}
}
// Include the bad image — different bad images on the same
// workload should be distinct Findings (operator just typo'd
// one container, the rest are fine). This also keeps per-image
// resolution after the job-family collapse above: one scan Job
// family with two bad images is still two Findings.
image := firstFailingImage(obj)
// The failing image discriminates between containers of one
// workload: an operator typo'd a single container and the rest are
// fine, so those stay distinct Findings.
//
// It is deliberately excluded for Job-owned Pods. For a
// one-Job-per-image creator (our image scanner) the image IS the
// per-run identity, so keeping it re-introduces exactly the fanout
// the job-family collapse just removed — verified on the dev
// cluster after #580 shipped: every scan Job still had its own
// fingerprint because each pulls a different image. The image stays
// in the evidence blocks either way; it just no longer forks the
// identity. Skipping the call also avoids walking every container
// status for a result we would discard.
var image string
if !jobOwned {
image = firstFailingImage(obj)
}
return fp("image_pull_backoff_reporter", ns, name, image)
},
EnrichBlocks: imagePullBackoffEnrichBlocks,
Expand Down
50 changes: 50 additions & 0 deletions runner/pkg/triggers/predicates_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -771,6 +771,56 @@ func TestImagePullBackoff_JobOwnedPodsCollapseToJobFamily(t *testing.T) {
}
}

func TestImagePullBackoff_JobFamilyCollapsesAcrossDifferentImages(t *testing.T) {
// Regression for what #580 missed: a one-Job-per-image creator gives every
// run a different image, so keeping the image in the fingerprint re-forked
// the identity even after the job-family collapse. Observed on the dev
// cluster — every trivy-image-scan Job still had its own fingerprint.
mk := func(job, image string) map[string]any {
return asObj(t, `{
"metadata":{
"name":"`+job+`-p9x2","namespace":"scan",
"ownerReferences":[{"kind":"Job","name":"`+job+`","controller":true}]
},
"status":{"containerStatuses":[
{"name":"scan","image":"`+image+`",
"state":{"waiting":{"reason":"ErrImagePull"}}}
]}
}`)
}
m := imagePullBackoffMatcher()
a := m.FingerprintFn(mk("trivy-image-scan-020c9475", "reg/nudgebee-ticket-server:2026-08-20T09-31-55_2a48d6b"))
b := m.FingerprintFn(mk("trivy-image-scan-9d22cbf9", "reg/nudgebee-llm-server:2026-08-20T09-12-54_b8f84d9"))
if a != b {
t.Error("scan Jobs of one family must share a fingerprint even when each pulls a different image")
}
// A different family in the same namespace stays distinct.
if c := m.FingerprintFn(mk("kube-bench-scan-177dde3a", "reg/kube-bench:v1")); c == a {
t.Error("a different job family must not collapse into the same fingerprint")
}
}

func TestImagePullBackoff_NonJobOwnersKeepImageDiscrimination(t *testing.T) {
// The image must still split Findings for ordinary workloads — one typo'd
// container should not hide behind a sibling's good image.
mk := func(image string) map[string]any {
return asObj(t, `{
"metadata":{
"name":"api-7f9d8c5b6d-aaaaa","namespace":"prod",
"ownerReferences":[{"kind":"ReplicaSet","name":"api-7f9d8c5b6d","controller":true}]
},
"status":{"containerStatuses":[
{"name":"app","image":"`+image+`",
"state":{"waiting":{"reason":"ImagePullBackOff"}}}
]}
}`)
}
m := imagePullBackoffMatcher()
if m.FingerprintFn(mk("registry/api:bad-v1")) == m.FingerprintFn(mk("registry/api:bad-v2")) {
t.Error("different bad images on a Deployment must stay distinct")
}
}

func TestImagePullBackoff_DeploymentReplicasStillCollapse(t *testing.T) {
// Guards the behaviour that already worked, so the job-family change
// above cannot regress it.
Expand Down
Loading