From 405a9cb1a59f59a8800091c8b012bfc9d3d623e1 Mon Sep 17 00:00:00 2001 From: Pascal Watteel Date: Sat, 7 Nov 2026 22:31:50 +0400 Subject: [PATCH] fix(parallel): size the process budget by the cgroup CPU quota runtime.NumCPU() counts host cores, not the container's quota, and go.mod's `go 1.23` keeps Go's container-aware GOMAXPROCS off. On the quadronix CI job pods (256 host cores, cpu.max 16 CPUs) the budget was 256, so the gate started ~350 prettier processes on 16 CPUs; they missed the 30 s formatter deadline and the Kuvryn nightly failed on files reported UNCHECKED. parallel.CPUs() now caps NumCPU by cgroup v2 cpu.max (or v1 cfs_quota/period), rounding a fractional quota up, at least one. The shared budget and the SAST slot count both use it. --- docs/domains.md | 7 +++- internal/parallel/cpus.go | 74 ++++++++++++++++++++++++++++++++++ internal/parallel/cpus_test.go | 59 +++++++++++++++++++++++++++ internal/parallel/parallel.go | 7 ++-- internal/security/security.go | 3 +- 5 files changed, 142 insertions(+), 8 deletions(-) create mode 100644 internal/parallel/cpus.go create mode 100644 internal/parallel/cpus_test.go diff --git a/docs/domains.md b/docs/domains.md index ed608f76..73b97abb 100644 --- a/docs/domains.md +++ b/docs/domains.md @@ -64,8 +64,11 @@ tree and filters the findings instead. Every scan is bounded so a machine running several agents is not saturated: semgrep runs with `--jobs=2`; one scan per repository and a -quarter of the cores' worth of scans per machine run at once, the rest -wait for a slot. The gate's whole SAST allowance — waiting and +quarter of the usable CPUs' worth of scans per machine run at once, the +rest wait for a slot. Usable CPUs are the cores capped by the cgroup CPU +quota, so a CI container that sees 256 host cores but may use 16 sizes +this, and every other procoder fan-out (formatters, secret scan, gate +legs), by the 16. The gate's whole SAST allowance — waiting and scanning — is 90 seconds when the commit hook runs it, under the hook's 120, and 10 minutes for a direct `procoder check` or CI. A scan that could not get a slot, or ran out of time, reports SAST as NOT run and blocks; diff --git a/internal/parallel/cpus.go b/internal/parallel/cpus.go new file mode 100644 index 00000000..bc617858 --- /dev/null +++ b/internal/parallel/cpus.go @@ -0,0 +1,74 @@ +package parallel + +import ( + "os" + "path/filepath" + "runtime" + "strconv" + "strings" +) + +// CPUs is how many CPUs this process may actually use: runtime.NumCPU() +// capped by the cgroup CPU quota, at least one. +// +// runtime.NumCPU() counts the host's cores (minus the affinity mask), not +// the container's quota, and go.mod's `go 1.23` keeps Go 1.25's +// container-aware GOMAXPROCS off. On a CI job pod that sees 256 cores but +// may use 16, sizing by NumCPU started hundreds of formatter processes, +// which then missed their deadline and were reported as not checked. +func CPUs() int { return cpusFrom("/sys/fs/cgroup", runtime.NumCPU()) } + +// cpusFrom caps host by the quota found under the cgroup root: cgroup v2's +// cpu.max, else cgroup v1's cpu.cfs_quota_us / cpu.cfs_period_us. A missing, +// unlimited or unreadable quota leaves host as it is. A fractional quota +// rounds up, so 1.5 CPUs allows two processes. +func cpusFrom(root string, host int) int { + host = maxInt(1, host) + quota, period, ok := cgroupV2(root) + if !ok { + quota, period, ok = cgroupV1(root) + } + if !ok || quota <= 0 || period <= 0 { + return host + } + n := int((quota + period - 1) / period) + if n < 1 { + n = 1 + } + if n < host { + return n + } + return host +} + +func cgroupV2(root string) (quota, period int64, ok bool) { + b, err := os.ReadFile(filepath.Join(root, "cpu.max")) + if err != nil { + return 0, 0, false + } + f := strings.Fields(string(b)) + if len(f) != 2 || f[0] == "max" { + return 0, 0, false + } + return parsePair(f[0], f[1]) +} + +func cgroupV1(root string) (quota, period int64, ok bool) { + for _, dir := range []string{"cpu", "cpu,cpuacct"} { + q, err1 := os.ReadFile(filepath.Join(root, dir, "cpu.cfs_quota_us")) + p, err2 := os.ReadFile(filepath.Join(root, dir, "cpu.cfs_period_us")) + if err1 == nil && err2 == nil { + return parsePair(strings.TrimSpace(string(q)), strings.TrimSpace(string(p))) + } + } + return 0, 0, false +} + +func parsePair(q, p string) (quota, period int64, ok bool) { + quota, err1 := strconv.ParseInt(q, 10, 64) + period, err2 := strconv.ParseInt(p, 10, 64) + if err1 != nil || err2 != nil || quota <= 0 || period <= 0 { + return 0, 0, false + } + return quota, period, true +} diff --git a/internal/parallel/cpus_test.go b/internal/parallel/cpus_test.go new file mode 100644 index 00000000..a810e596 --- /dev/null +++ b/internal/parallel/cpus_test.go @@ -0,0 +1,59 @@ +package parallel + +import ( + "os" + "path/filepath" + "testing" +) + +// A CI job container can see every core of its host but be allowed far +// fewer: the arc runners on quadronix show runtime.NumCPU() = 256 under a +// cgroup cpu.max of 16 CPUs. A budget of 256 started ~350 prettier +// processes on 16 CPUs and every one missed its 30-second deadline, so the +// gate reported the files as UNCHECKED. The budget has to follow the quota. +// +// proved by: return host from cpusFrom unchanged — the quota cases fail. +func TestCPUsFromFollowsTheCgroupQuota(t *testing.T) { + cases := []struct { + name string + cpuMax string // "" = no cgroup v2 file + v1 [2]string + host int + want int + }{ + {name: "quadronix runner", cpuMax: "1600000 100000\n", host: 256, want: 16}, + {name: "fractional quota rounds up", cpuMax: "150000 100000\n", host: 8, want: 2}, + {name: "unlimited", cpuMax: "max 100000\n", host: 10, want: 10}, + {name: "quota above host", cpuMax: "6400000 100000\n", host: 4, want: 4}, + {name: "no cgroup", host: 6, want: 6}, + {name: "garbage", cpuMax: "lots\n", host: 6, want: 6}, + {name: "cgroup v1 quota", v1: [2]string{"400000\n", "100000\n"}, host: 64, want: 4}, + {name: "cgroup v1 unlimited", v1: [2]string{"-1\n", "100000\n"}, host: 64, want: 64}, + {name: "tiny quota still one", cpuMax: "1000 100000\n", host: 8, want: 1}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + root := t.TempDir() + if c.cpuMax != "" { + write(t, filepath.Join(root, "cpu.max"), c.cpuMax) + } + if c.v1[0] != "" { + write(t, filepath.Join(root, "cpu", "cpu.cfs_quota_us"), c.v1[0]) + write(t, filepath.Join(root, "cpu", "cpu.cfs_period_us"), c.v1[1]) + } + if got := cpusFrom(root, c.host); got != c.want { + t.Errorf("cpusFrom(%q, %d) = %d, want %d", c.name, c.host, got, c.want) + } + }) + } +} + +func write(t *testing.T, path, body string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } +} diff --git a/internal/parallel/parallel.go b/internal/parallel/parallel.go index fff91f87..e0cae3a0 100644 --- a/internal/parallel/parallel.go +++ b/internal/parallel/parallel.go @@ -24,17 +24,16 @@ // and fixed in number. package parallel -import "runtime" - // budget is the number of units of work that may be in flight. Sized to // the machine, at least one — a zero-capacity channel would deadlock, and // runtime.NumCPU has returned 0 on constrained containers. var budget = make(chan struct{}, size()) -// size is how many units may run at once. At least one: runtime.NumCPU has +// size is how many units may run at once: the CPUs this process may use, +// cgroup quota included (see CPUs). At least one: runtime.NumCPU has // returned 0 on constrained containers, and a zero-capacity channel would // deadlock the gate rather than slow it. -func size() int { return maxInt(1, runtime.NumCPU()) } +func size() int { return maxInt(1, CPUs()) } func maxInt(a, b int) int { if a > b { diff --git a/internal/security/security.go b/internal/security/security.go index ae0d03d2..14e98899 100644 --- a/internal/security/security.go +++ b/internal/security/security.go @@ -18,7 +18,6 @@ import ( "path" "path/filepath" "regexp" - "runtime" "sort" "strconv" "strings" @@ -402,7 +401,7 @@ const ( // sastSlots is how many scans may run at once on this machine, across // every repository and every procoder process: a quarter of the cores, // each scan using sastJobs of them. -var sastSlots = max(1, runtime.NumCPU()/4) +var sastSlots = max(1, parallel.CPUs()/4) // sastSlotDir is where the slots live; a variable so tests can use their // own.