From fabfb34613248a7bbf1dc2157d7b7052b608c67a Mon Sep 17 00:00:00 2001 From: shiv Date: Fri, 14 Aug 2026 20:28:39 +0530 Subject: [PATCH 1/2] perf(discovery): pre-parse and cache whenExpr in Collector during Pack validation --- .jules/bolt.md | 4 ++++ pkg/proxy/discovery/example_pack_test.go | 18 ++++++++++++++- pkg/proxy/discovery/pack.go | 28 ++++++++++++++++-------- pkg/proxy/discovery/pack_test.go | 17 +++++++++++++- 4 files changed, 56 insertions(+), 11 deletions(-) diff --git a/.jules/bolt.md b/.jules/bolt.md index 7f00bcc..5a906d2 100644 --- a/.jules/bolt.md +++ b/.jules/bolt.md @@ -9,3 +9,7 @@ Critical learnings and performance patterns discovered in this codebase. ## 2026-08-13 - Use Switch Statements for Zero-Allocation Static Lookups **Learning:** Defining static lookup map literals (such as `map[string]string{...}`) inside helper functions evaluated per-host (e.g., `osFamily` in `parseFacts`) causes Go to allocate and populate a new hash map on the heap on every invocation (~1.2 KB and 3 allocations per call). **Action:** Prefer switch statements over map literals for fixed static lookups to achieve zero heap allocations, complete immutability, and zero race-condition risk. + +## 2026-08-14 - Pre-parse and Cache Guard Expressions at Pack Validation +**Learning:** Re-parsing filter expressions or guard ASTs (such as `when` expressions in content packs) inside per-target evaluation loops (`Select`) during fleet-wide sweeps causes massive repeated string splitting, heap allocations, and CPU overhead ($O(N_{\text{hosts}} \cdot N_{\text{collectors}})$ parses). +**Action:** Parse guard expressions once into compiled AST structs (`*whenExpr`) during pack load/validation (`Pack.validate`), caching the pointer on the collector struct for direct zero-parse evaluations. diff --git a/pkg/proxy/discovery/example_pack_test.go b/pkg/proxy/discovery/example_pack_test.go index 017fc66..5756bd8 100644 --- a/pkg/proxy/discovery/example_pack_test.go +++ b/pkg/proxy/discovery/example_pack_test.go @@ -88,7 +88,7 @@ func TestExamplePackGuardsModularityTagForOldRpm(t *testing.T) { } } -func loadExamplePack(t *testing.T) *Pack { +func loadExamplePack(t testing.TB) *Pack { t.Helper() raw, err := os.ReadFile("../../../docs/content-packs/linux-inventory-example.yaml") if err != nil { @@ -118,3 +118,19 @@ func collectorCmd(t *testing.T, pack *Pack, id string) string { } func b64(b []byte) string { return base64.StdEncoding.EncodeToString(b) } + +func BenchmarkExamplePackSelect(b *testing.B) { + pack := loadExamplePack(b) + facts := map[string]string{ + "os_family": "rhel", + "os_id": "rocky", + "os_major": "9", + "arch": "x86_64", + } + + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + _, _ = pack.Select(facts) + } +} diff --git a/pkg/proxy/discovery/pack.go b/pkg/proxy/discovery/pack.go index 414c623..6c1a6b8 100644 --- a/pkg/proxy/discovery/pack.go +++ b/pkg/proxy/discovery/pack.go @@ -30,6 +30,8 @@ type Collector struct { ID string `yaml:"id"` When string `yaml:"when,omitempty"` // empty = run on every host Cmd string `yaml:"cmd"` + + expr *whenExpr // parsed once at pack validation time to avoid per-host re-parsing } // KindInventory is the only pack kind Phase 0 executes. @@ -152,7 +154,8 @@ func (p *Pack) validate() error { } seen := make(map[string]bool, len(p.Collectors)) - for i, c := range p.Collectors { + for i := range p.Collectors { + c := &p.Collectors[i] if c.ID == "" { return fmt.Errorf("collector %d has no id", i) } @@ -165,11 +168,14 @@ func (p *Pack) validate() error { return fmt.Errorf("collector %q has no cmd", c.ID) } // Reject unparseable guards at load time rather than per host, so a - // malformed pack fails once and loudly. + // malformed pack fails once and loudly. Cache the parsed expression to + // avoid re-parsing guards on every host during inventory sweeps. if c.When != "" { - if _, err := parseWhen(c.When); err != nil { + expr, err := parseWhen(c.When) + if err != nil { return fmt.Errorf("collector %q: %w", c.ID, err) } + c.expr = expr } } return nil @@ -179,7 +185,7 @@ func (p *Pack) validate() error { // in pack order. A guard referencing an unknown fact does not match — see // evalWhen. func (p *Pack) Select(facts map[string]string) ([]Collector, []SkippedCollector) { - var run []Collector + run := make([]Collector, 0, len(p.Collectors)) var skipped []SkippedCollector for _, c := range p.Collectors { @@ -187,11 +193,15 @@ func (p *Pack) Select(facts map[string]string) ([]Collector, []SkippedCollector) run = append(run, c) continue } - expr, err := parseWhen(c.When) - if err != nil { - // validate() already rejected these; defensive. - skipped = append(skipped, SkippedCollector{ID: c.ID, Reason: err.Error()}) - continue + expr := c.expr + if expr == nil { + var err error + expr, err = parseWhen(c.When) + if err != nil { + // validate() already rejected these; defensive. + skipped = append(skipped, SkippedCollector{ID: c.ID, Reason: err.Error()}) + continue + } } match, err := expr.eval(facts) if err != nil { diff --git a/pkg/proxy/discovery/pack_test.go b/pkg/proxy/discovery/pack_test.go index 33de6f5..df747bf 100644 --- a/pkg/proxy/discovery/pack_test.go +++ b/pkg/proxy/discovery/pack_test.go @@ -9,7 +9,7 @@ import ( // signPack produces a signed pack document from a body (a pack without its // signature line), mirroring what the publish pipeline will do in CI. -func signPack(t *testing.T, body string, priv ed25519.PrivateKey) string { +func signPack(t testing.TB, body string, priv ed25519.PrivateKey) string { t.Helper() sig := ed25519.Sign(priv, SignedBytes([]byte(body))) return body + "\nsignature: " + base64.StdEncoding.EncodeToString(sig) + "\n" @@ -334,3 +334,18 @@ func BenchmarkParseAndVerify(b *testing.B) { _, _ = ParseAndVerify(doc, pub) } } + +func BenchmarkPackSelect(b *testing.B) { + pub, priv := testKeys(b) + pack, err := ParseAndVerify([]byte(signPack(b, validBody, priv)), pub) + if err != nil { + b.Fatalf("verifying pack: %v", err) + } + facts := map[string]string{"os_family": "debian", "arch": "x86_64", "os_id": "ubuntu", "os_major": "22"} + + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + _, _ = pack.Select(facts) + } +} From 4f35e153ca6699968eb4d4016379b2469b013f22 Mon Sep 17 00:00:00 2001 From: shiv Date: Fri, 14 Aug 2026 21:41:19 +0530 Subject: [PATCH 2/2] build(deps): bump go toolchain to 1.25.13 to resolve stdlib vulnerabilities --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 683fad8..d73015c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module nudgebee/forager -go 1.25.12 +go 1.25.13 require ( cloud.google.com/go/auth v0.22.0