diff --git a/execution/cache/cache_test.go b/execution/cache/cache_test.go index f0d8a7e02cd..0a5e94f3bdb 100644 --- a/execution/cache/cache_test.go +++ b/execution/cache/cache_test.go @@ -20,6 +20,7 @@ import ( "bytes" "encoding/binary" "fmt" + "slices" "sync" "sync/atomic" "testing" @@ -1771,3 +1772,102 @@ func BenchmarkPublishVsViewBindLock(b *testing.B) { }) } } + +// The O(1) entry counter that replaced freelru's all-shard Len on the grow +// check must not drift from the LRU's real length on any mutation path. +func TestGenericCache_LenTracksLRU(t *testing.T) { + c := closeOnCleanup(t, NewGenericCacheWithAvg[[]byte](8*datasize.MB, 256, func(v []byte) int { return len(v) }, ModeEvictLRU)) + key := func(i int) []byte { + k := make([]byte, 8) + binary.BigEndian.PutUint64(k, uint64(i)) + return k + } + lruLen := func() int { + g := c.data.Load() + sum := 0 + for i := range g.shards { + sum += g.shards[i].Len() + } + return sum + } + check := func(phase string) { + t.Helper() + require.Equal(t, lruLen(), c.Len(), "entry counter drifted after %s", phase) + } + + for i := range 500 { + c.Put(key(i), []byte("v"), 10) + } + check("inserts") + + for i := range 200 { + c.Put(key(i), []byte("updated"), 20) + } + check("updates") + + for i := range 100 { + c.Delete(key(i)) + } + check("deletes") + + // Floor 15 leaves the txNum-10 entries live and strands the txNum-20 ones, + // which their next read drops. + c.Unwind(15) + for i := 100; i < 500; i++ { + c.Get(key(i)) + } + check("stale drops") + + for i := 500; i < 4000; i++ { + c.Put(key(i), []byte("v"), 30) + } + g := c.data.Load() + require.Greater(t, slices.Max(g.curCap), g.startCapPerShard, "no shard grew") + check("grow") + + c.Clear() + require.Equal(t, 0, c.Len()) + check("clear") +} + +// Storing a key the generation already holds must not raise the count: freelru +// replaces it in place and fires no OnEvict, so the rise never comes back down +// and the grow gate trips on entries that are not there. +func TestGrowLRU_PutOfPresentKeyKeepsCount(t *testing.T) { + t.Run("same key twice", func(t *testing.T) { + g := newGrowLRU[int](1*datasize.MB, 8, nil) + defer g.Close() + + g.Put(1, 10) + g.Put(1, 20) + + require.Equal(t, 1, g.Len()) + v, ok := g.Get(1) + require.True(t, ok) + require.Equal(t, 20, v) + }) + + // A grow copies a key into the next generation, a read-path Remove then + // drops it from the still-current old one, and a writer that missed on old + // stores it after the publish -- into a generation that already holds it. + t.Run("key carried in by a racing grow copy", func(t *testing.T) { + var evicted int + g := newGrowLRU[int](1*datasize.MB, 8, func(uint64, int) { evicted++ }) + defer g.Close() + + g.Put(1, 10) + old := g.cur.Load() + + next := g.newShards(g.curCap.Load()) + next.add(1, 10) // the grow's copy + old.lru.Remove(1) + g.cur.Store(next) // the publish the writer's miss straddles + + evictedBefore := evicted + g.Put(1, 20) + + require.Equal(t, 1, g.Len()) + require.Equal(t, 1, evicted-evictedBefore, + "the copy the store displaced must be evicted, not dropped silently") + }) +} diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index 881eaceeaac..8504a506f31 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -191,11 +191,10 @@ type CodeCache struct { // putContentLocked is the shared insert path for the content-addressed code layers // (hashToCode, codeHashToCode, codeSizeByCodeHash). Each is a freelru.ShardedLRU // of per-key-immutable entries carrying a (txNum, epoch) stamp: a live entry is -// kept (its bytes/size are invariant for a given key), a stale one is removed -// (its OnEvict decrements counter) so the fresh entry can replace it, and once -// the entry-count cap is reached freelru.Add evicts the coldest entry (whose -// OnEvict decrements counter) rather than freezing. counter tracks resident -// bytes as a stat; the hard bound is the LRU's entry cap. stamp/valCost are +// kept (its bytes/size are invariant for a given key), and whatever Put displaces +// — a stale entry, or the coldest one once the cap is reached — decrements +// counter through OnEvict. counter tracks resident bytes as a stat; the hard +// bound is the LRU's entry cap. stamp/valCost are // non-capturing so passing them allocates nothing on the put path. The caller // holds the key's put stripe. func putContentLocked[T any]( @@ -212,10 +211,9 @@ func putContentLocked[T any]( if txNum, epoch := stamp(existing); !coh.IsStale(txNum, epoch) { return } - lru.Remove(h) // stale — OnEvict decrements counter for the removed entry } counter.Add(keyCost + valCost(newEntry)) - lru.Add(h, newEntry) // evicts the coldest entry when full; its OnEvict decrements counter + lru.Put(h, newEntry) } func codeEntryStamp(e codeEntry) (uint64, uint32) { return e.txNum, e.epoch } diff --git a/execution/cache/code_cache_concurrency_test.go b/execution/cache/code_cache_concurrency_test.go index 88a9c3d3fa1..06b6d8920f4 100644 --- a/execution/cache/code_cache_concurrency_test.go +++ b/execution/cache/code_cache_concurrency_test.go @@ -20,6 +20,7 @@ import ( "encoding/binary" "runtime" "sync" + "sync/atomic" "testing" "github.com/c2h5oh/datasize" @@ -80,7 +81,7 @@ func TestCodeCache_ByteCheckRejectsForeignKeyHash(t *testing.T) { foreign := make([]byte, 32) copy(foreign, realHash) foreign[0] ^= 0xff // different 32-byte key - cc.codeHashToCode.Add(maphash.Hash(foreign), codeEntry{code: code, keyHash: hash32(realHash), txNum: 1, epoch: cc.coh.Epoch()}) + cc.codeHashToCode.Put(maphash.Hash(foreign), codeEntry{code: code, keyHash: hash32(realHash), txNum: 1, epoch: cc.coh.Epoch()}) // The stored entry's keyHash is realHash, not foreign — Get must reject it. _, ok = cc.GetByCodeHash(foreign) @@ -150,7 +151,7 @@ func TestCodeCache_ClearRacingPut_EpochAlias(t *testing.T) { // Model a writer that sampled the epoch before Clear and published after // the relevant layers were purged. cc.addrToHash.Add(common.BytesToAddress(addr), versionedAddressID{addrID: codeID, txNum: 200, epoch: preClearEpoch}) - cc.hashToCode.Add(codeID, codeEntry{code: code, txNum: 200, epoch: preClearEpoch}) + cc.hashToCode.Put(codeID, codeEntry{code: code, txNum: 200, epoch: preClearEpoch}) cc.Unwind(150) _, ok := cc.Get(addr) @@ -193,3 +194,196 @@ func TestCodeCache_ClearFencesStartedPut(t *testing.T) { _, ok := cc.Get(addr) require.False(t, ok, "Clear must remove a write that started in the retiring generation") } + +// The O(1) entry counter that replaced freelru's all-shard Len on growLRU's add +// path must not drift from the LRU's real length on any mutation path. +func TestGrowLRU_LenTracksLRU(t *testing.T) { + var evictions int + g := newGrowLRU[uint64](8*datasize.MB, 16, func(uint64, uint64) { evictions++ }) + defer g.Close() + check := func(phase string) { + t.Helper() + require.Equal(t, g.cur.Load().lru.Len(), g.Len(), "entry counter drifted after %s", phase) + } + // growLRU hashes keys with the identity function, so spread the shard-selection + // bits the way its real maphash/keccak-derived keys do. + key := func(i uint64) uint64 { return i * 0x9E3779B97F4A7C15 } + + for i := range uint64(500) { + g.Put(key(i), i) + } + check("adds") + require.Equal(t, 500, g.Len(), "500 distinct keys must fit below the 1024-slot start capacity") + + for i := range uint64(100) { + g.Remove(key(i)) + } + check("removes") + require.Equal(t, 100, evictions, "the caller's OnEvict must still fire for each removal") + + before := g.cur.Load() + for i := uint64(500); i < 4000; i++ { + g.Put(key(i), i) + } + require.NotEqual(t, before, g.cur.Load(), "grow did not happen") + check("grow") + + g.Purge() + require.Equal(t, 0, g.Len()) + check("purge") +} + +// A grow can copy a key into the new generation while a same-key refresh is in +// flight. Put resolves g.cur once so its removal and its store land on the same +// generation; split across the swap, the store would replace the copy in place +// without firing OnEvict and double-count it. +func TestGrowLRU_GrowRacePutDoesNotDoubleCount(t *testing.T) { + g := newGrowLRU[uint64](8*datasize.MB, 16, nil) + defer g.Close() + + h := uint64(7) + g.Put(h, 1) + gen1 := g.cur.Load() + + // Build the next generation exactly like maybeGrow's copy loop, and + // publish it while h is still present in gen1. + newCap := g.curCap.Load() * genericCacheGrowFactor + gen2 := g.newShards(newCap) + for _, k := range gen1.lru.Keys() { + if v, ok := gen1.lru.Get(k); ok { + gen2.add(k, v) + } + } + g.cur.Store(gen2) + g.curCap.Store(newCap) + + g.Put(h, 2) + + require.Equal(t, gen2.lru.Len(), g.Len(), + "counter must not double-count a grow-copied key replaced in place") + got, ok := g.Get(h) + require.True(t, ok) + require.Equal(t, uint64(2), got, "the refreshed value must be the one served") + + // A refresh that resolved the retired generation must leave the live count alone. + live := g.Len() + gen1.lru.Remove(h) + gen1.add(h, 3) + require.Equal(t, live, g.Len(), "a write lost in the retired generation must not move the live counter") + require.Equal(t, gen1.lru.Len(), gen1.len(), "the retired generation's own counter must stay exact") +} + +// Each layer's counter must equal its LRU's real length while all three are +// driven through putContentLocked's displacing path. The fixture repeats keys +// and keeps unwinding them so that path runs, with budget left to grow into. +func TestCodeCache_GrowLRULenCounterUnderConcurrency(t *testing.T) { + // 64MB over avgCodeEntryBytes puts the two code layers' ceiling well above + // genericCacheStartCapacity; at 8MB it lands below and they never grow. + cc := closeOnCleanup(t, NewCodeCache(64*datasize.MB, 8*datasize.MB)) + // Floor 0, so every later Unwind turns the whole resident set stale on its + // epoch bump alone. + cc.Unwind(0) + + const workers = 8 + const perWorker = 3000 + const keySpace = 4096 + + var unwinder sync.WaitGroup + done := make(chan struct{}) + unwinder.Go(func() { + for { + select { + case <-done: + return + default: + } + cc.Unwind(0) + runtime.Gosched() + } + }) + + var wg sync.WaitGroup + for w := range workers { + wg.Go(func() { + addr := make([]byte, 20) + code := make([]byte, 40) + codeHash := make([]byte, 32) + for i := range perWorker { + k := uint64(w*perWorker+i) % keySpace + binary.BigEndian.PutUint64(addr[12:], k) + binary.BigEndian.PutUint64(code, k) + binary.BigEndian.PutUint64(codeHash, k) + cc.PutWithCodeHash(addr, code, codeHash, uint64(i)) + } + }) + } + wg.Wait() + close(done) + unwinder.Wait() + + for _, layer := range []struct { + name string + lru *growLRU[codeEntry] + }{ + {"hashToCode", cc.hashToCode}, + {"codeHashToCode", cc.codeHashToCode}, + } { + require.Equal(t, layer.lru.cur.Load().lru.Len(), layer.lru.Len(), "%s counter drifted", layer.name) + require.Greater(t, layer.lru.curCap.Load(), layer.lru.startCap, "%s never grew", layer.name) + } + require.Equal(t, cc.codeSizeByCodeHash.cur.Load().lru.Len(), cc.codeSizeByCodeHash.Len(), + "codeSizeByCodeHash counter drifted") + require.Greater(t, cc.codeSizeByCodeHash.curCap.Load(), cc.codeSizeByCodeHash.startCap, + "codeSizeByCodeHash never grew") +} + +// Parallel adds while the LRU is still below its ceiling — the window where +// every add runs the grow check. +func BenchmarkGrowLRUParallelPutGrow(b *testing.B) { + g := newGrowLRU[uint64](256*datasize.MB, 48, nil) + defer g.Close() + var seq atomic.Uint64 + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + n := seq.Add(1) * 0x9E3779B97F4A7C15 + g.Put(n, n) + } + }) +} + +// CodeCache's read-path stale drops hold no put stripe, so they interleave +// freely with a striped refresh while the LRU grows. The counter must still +// equal the real length: both grow gates read it, so an under-count wedges +// growth for good. +func TestGrowLRU_CountExactUnderStripedRefreshAndUnstripedRemove(t *testing.T) { + g := newGrowLRU[uint64](8*datasize.MB, 16, nil) + defer g.Close() + + const keySpace = 4096 + key := func(i uint64) uint64 { return (i % keySpace) * 0x9E3779B97F4A7C15 } + + var stripes [256]sync.Mutex + var wg sync.WaitGroup + for w := range 8 { + wg.Go(func() { + for i := range uint64(20000) { + h := key(uint64(w)*20000 + i) + stripe := &stripes[uint8(h)] + stripe.Lock() + g.Put(h, i) + stripe.Unlock() + } + }) + } + wg.Go(func() { + for i := range uint64(60000) { + g.Remove(key(i)) + } + }) + wg.Wait() + + gen := g.cur.Load() + require.Equal(t, gen.lru.Len(), gen.len(), "live generation's counter drifted from its real length") + require.NotEqual(t, g.startCap, g.curCap.Load(), "the LRU never grew, so the growth gates were not exercised") +} diff --git a/execution/cache/grow_lru.go b/execution/cache/grow_lru.go index b0e3725028d..3eca24fecf2 100644 --- a/execution/cache/grow_lru.go +++ b/execution/cache/grow_lru.go @@ -27,6 +27,24 @@ import ( "github.com/erigontech/erigon/common/cachebudget" ) +// lruGen is one generation of a sharded LRU plus an O(1) live-entry count. +// freelru's own Len locks every shard, which serialises readers on a path that +// only wanted a number. +type lruGen[V any] struct { + lru *freelru.ShardedLRU[uint64, V] + n atomic.Int64 +} + +func (g *lruGen[V]) len() int { return int(g.n.Load()) } + +// add requires a key the LRU does not hold — Put removes first — so the count +// can rise unconditionally; a capacity eviction fires OnEvict, which lowers it. +func (g *lruGen[V]) add(h uint64, v V) (evicted bool) { + evicted = g.lru.Add(h, v) + g.n.Add(1) + return evicted +} + // growLRU is a uint64-keyed sharded LRU that starts small and jump-resizes ×4 // toward a byte-budget ceiling as it fills, funding each step from the shared // cachebudget envelope. It exists so a cache with a small working set never @@ -38,11 +56,10 @@ import ( // write lost in a retired generation is a benign miss, and an entry whose // removal a racing copy undid serves correct bytes until its stale stamp // drops it on the next read. Do not reuse for mutable-per-key values — those -// need GenericCache's fenced swap. The onEvict-maintained counters are -// approximate across grow windows (a lost write is counted but never -// evicted; a raced removal can subtract twice). +// need GenericCache's fenced swap. Each generation's own entry count is exact; +// the caller's onEvict byte counters stay approximate across grow windows. type growLRU[V any] struct { - cur atomic.Pointer[freelru.ShardedLRU[uint64, V]] + cur atomic.Pointer[lruGen[V]] onEvict func(uint64, V) avgBytes int64 @@ -71,26 +88,35 @@ func newGrowLRU[V any](maxBytes datasize.ByteSize, avgBytes uint32, onEvict func return g } -func (g *growLRU[V]) newShards(capacity uint32) *freelru.ShardedLRU[uint64, V] { +func (g *growLRU[V]) newShards(capacity uint32) *lruGen[V] { lru, err := freelru.NewSharded[uint64, V](capacity, u64identity) if err != nil { panic(fmt.Sprintf("growLRU: NewSharded(%d): %s", capacity, err)) } - if g.onEvict != nil { - lru.SetOnEvict(g.onEvict) - } - return lru + gen := &lruGen[V]{lru: lru} + lru.SetOnEvict(func(k uint64, v V) { + gen.n.Add(-1) + if g.onEvict != nil { + g.onEvict(k, v) + } + }) + return gen } -func (g *growLRU[V]) Get(key uint64) (V, bool) { return g.cur.Load().Get(key) } +func (g *growLRU[V]) Get(key uint64) (V, bool) { return g.cur.Load().lru.Get(key) } -func (g *growLRU[V]) Add(key uint64, value V) { - lru := g.cur.Load() - if curCap := g.curCap.Load(); curCap < g.maxCap && lru.Len() >= int(curCap) { +// Put stores a key, growing first when the generation is full. The Remove is not +// redundant after a caller's miss: an unfenced grow can leave the key present in +// the generation this Put lands on, and freelru's Add would replace it in place +// without firing OnEvict, stranding the count and the caller's byte counter. +func (g *growLRU[V]) Put(key uint64, value V) { + gen := g.cur.Load() + if curCap := g.curCap.Load(); curCap < g.maxCap && gen.len() >= int(curCap) { g.maybeGrow() - lru = g.cur.Load() + gen = g.cur.Load() } - lru.Add(key, value) + gen.lru.Remove(key) + gen.add(key, value) } func (g *growLRU[V]) maybeGrow() { @@ -98,7 +124,7 @@ func (g *growLRU[V]) maybeGrow() { defer g.resizeMu.Unlock() old := g.cur.Load() curCap := g.curCap.Load() - if curCap >= g.maxCap || old.Len() < int(curCap) { + if curCap >= g.maxCap || old.len() < int(curCap) { return } newCap := min(curCap*genericCacheGrowFactor, g.maxCap) @@ -107,9 +133,9 @@ func (g *growLRU[V]) maybeGrow() { return } next := g.newShards(newCap) - for _, k := range old.Keys() { - if v, ok := old.Get(k); ok { - next.Add(k, v) + for _, k := range old.lru.Keys() { + if v, ok := old.lru.Get(k); ok { + next.add(k, v) } } g.cur.Store(next) @@ -117,8 +143,8 @@ func (g *growLRU[V]) maybeGrow() { g.reserved += delta } -func (g *growLRU[V]) Remove(key uint64) { g.cur.Load().Remove(key) } -func (g *growLRU[V]) Len() int { return g.cur.Load().Len() } +func (g *growLRU[V]) Remove(key uint64) { g.cur.Load().lru.Remove(key) } +func (g *growLRU[V]) Len() int { return g.cur.Load().len() } // Purge empties the LRU and shrinks it back to the start size, returning the // grown budget to the envelope (it regrows on demand).