execution/cache: track GenericCache entry count in an atomic, not freelru's all-shard Len

execution/cache/cache_test.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"encoding/binary"
 	"fmt"
+	"slices"
 	"sync"
 	"sync/atomic"
 	"testing"
@@ -1771,3 +1772,102 @@ func BenchmarkPublishVsViewBindLock(b *testing.B) {
 		})
 	}
 }
+
+// The O(1) entry counter that replaced freelru's all-shard Len on the grow
+// check must not drift from the LRU's real length on any mutation path.
+func TestGenericCache_LenTracksLRU(t *testing.T) {
+	c := closeOnCleanup(t, NewGenericCacheWithAvg[[]byte](8*datasize.MB, 256, func(v []byte) int { return len(v) }, ModeEvictLRU))
+	key := func(i int) []byte {
+		k := make([]byte, 8)
+		binary.BigEndian.PutUint64(k, uint64(i))
+		return k
+	}
+	lruLen := func() int {
+		g := c.data.Load()
+		sum := 0
+		for i := range g.shards {
+			sum += g.shards[i].Len()
+		}
+		return sum
+	}
+	check := func(phase string) {
+		t.Helper()
+		require.Equal(t, lruLen(), c.Len(), "entry counter drifted after %s", phase)
+	}
+
+	for i := range 500 {
+		c.Put(key(i), []byte("v"), 10)
+	}
+	check("inserts")
+
+	for i := range 200 {
+		c.Put(key(i), []byte("updated"), 20)
+	}
+	check("updates")
+
+	for i := range 100 {
+		c.Delete(key(i))
+	}
+	check("deletes")
+
+	// Floor 15 leaves the txNum-10 entries live and strands the txNum-20 ones,
+	// which their next read drops.
+	c.Unwind(15)
+	for i := 100; i < 500; i++ {
+		c.Get(key(i))
+	}
+	check("stale drops")
+
+	for i := 500; i < 4000; i++ {
+		c.Put(key(i), []byte("v"), 30)
+	}
+	g := c.data.Load()
+	require.Greater(t, slices.Max(g.curCap), g.startCapPerShard, "no shard grew")
+	check("grow")
+
+	c.Clear()
+	require.Equal(t, 0, c.Len())
+	check("clear")
+}
+
+// Storing a key the generation already holds must not raise the count: freelru
+// replaces it in place and fires no OnEvict, so the rise never comes back down
+// and the grow gate trips on entries that are not there.
+func TestGrowLRU_PutOfPresentKeyKeepsCount(t *testing.T) {
+	t.Run("same key twice", func(t *testing.T) {
+		g := newGrowLRU[int](1*datasize.MB, 8, nil)
+		defer g.Close()
+
+		g.Put(1, 10)
+		g.Put(1, 20)
+
+		require.Equal(t, 1, g.Len())
+		v, ok := g.Get(1)
+		require.True(t, ok)
+		require.Equal(t, 20, v)
+	})
+
+	// A grow copies a key into the next generation, a read-path Remove then
+	// drops it from the still-current old one, and a writer that missed on old
+	// stores it after the publish -- into a generation that already holds it.
+	t.Run("key carried in by a racing grow copy", func(t *testing.T) {
+		var evicted int
+		g := newGrowLRU[int](1*datasize.MB, 8, func(uint64, int) { evicted++ })
+		defer g.Close()
+
+		g.Put(1, 10)
+		old := g.cur.Load()
+
+		next := g.newShards(g.curCap.Load())
+		next.add(1, 10) // the grow's copy
+		old.lru.Remove(1)
+		g.cur.Store(next) // the publish the writer's miss straddles
+
+		evictedBefore := evicted
+		g.Put(1, 20)
+
+		require.Equal(t, 1, g.Len())
+		require.Equal(t, 1, evicted-evictedBefore,
+			"the copy the store displaced must be evicted, not dropped silently")
+	})
+}

execution/cache/code_cache.go

@@ -191,11 +191,10 @@ type CodeCache struct {
 // putContentLocked is the shared insert path for the content-addressed code layers
 // (hashToCode, codeHashToCode, codeSizeByCodeHash). Each is a freelru.ShardedLRU
 // of per-key-immutable entries carrying a (txNum, epoch) stamp: a live entry is
-// kept (its bytes/size are invariant for a given key), a stale one is removed
-// (its OnEvict decrements counter) so the fresh entry can replace it, and once
-// the entry-count cap is reached freelru.Add evicts the coldest entry (whose
-// OnEvict decrements counter) rather than freezing. counter tracks resident
-// bytes as a stat; the hard bound is the LRU's entry cap. stamp/valCost are
+// kept (its bytes/size are invariant for a given key), and whatever Put displaces
+// — a stale entry, or the coldest one once the cap is reached — decrements
+// counter through OnEvict. counter tracks resident bytes as a stat; the hard
+// bound is the LRU's entry cap. stamp/valCost are
 // non-capturing so passing them allocates nothing on the put path. The caller
 // holds the key's put stripe.
 func putContentLocked[T any](
@@ -212,10 +211,9 @@ func putContentLocked[T any](
 		if txNum, epoch := stamp(existing); !coh.IsStale(txNum, epoch) {
 			return
 		}
-		lru.Remove(h) // stale — OnEvict decrements counter for the removed entry
 	}
 	counter.Add(keyCost + valCost(newEntry))
-	lru.Add(h, newEntry) // evicts the coldest entry when full; its OnEvict decrements counter
+	lru.Put(h, newEntry)
 }
 
 func codeEntryStamp(e codeEntry) (uint64, uint32)         { return e.txNum, e.epoch }

execution/cache/code_cache_concurrency_test.go

@@ -20,6 +20,7 @@ import (
 	"encoding/binary"
 	"runtime"
 	"sync"
+	"sync/atomic"
 	"testing"
 
 	"github.com/c2h5oh/datasize"
@@ -80,7 +81,7 @@ func TestCodeCache_ByteCheckRejectsForeignKeyHash(t *testing.T) {
 	foreign := make([]byte, 32)
 	copy(foreign, realHash)
 	foreign[0] ^= 0xff // different 32-byte key
-	cc.codeHashToCode.Add(maphash.Hash(foreign), codeEntry{code: code, keyHash: hash32(realHash), txNum: 1, epoch: cc.coh.Epoch()})
+	cc.codeHashToCode.Put(maphash.Hash(foreign), codeEntry{code: code, keyHash: hash32(realHash), txNum: 1, epoch: cc.coh.Epoch()})
 
 	// The stored entry's keyHash is realHash, not foreign — Get must reject it.
 	_, ok = cc.GetByCodeHash(foreign)
@@ -150,7 +151,7 @@ func TestCodeCache_ClearRacingPut_EpochAlias(t *testing.T) {
 	// Model a writer that sampled the epoch before Clear and published after
 	// the relevant layers were purged.
 	cc.addrToHash.Add(common.BytesToAddress(addr), versionedAddressID{addrID: codeID, txNum: 200, epoch: preClearEpoch})
-	cc.hashToCode.Add(codeID, codeEntry{code: code, txNum: 200, epoch: preClearEpoch})
+	cc.hashToCode.Put(codeID, codeEntry{code: code, txNum: 200, epoch: preClearEpoch})
 	cc.Unwind(150)
 
 	_, ok := cc.Get(addr)
@@ -193,3 +194,196 @@ func TestCodeCache_ClearFencesStartedPut(t *testing.T) {
 	_, ok := cc.Get(addr)
 	require.False(t, ok, "Clear must remove a write that started in the retiring generation")
 }
+
+// The O(1) entry counter that replaced freelru's all-shard Len on growLRU's add
+// path must not drift from the LRU's real length on any mutation path.
+func TestGrowLRU_LenTracksLRU(t *testing.T) {
+	var evictions int
+	g := newGrowLRU[uint64](8*datasize.MB, 16, func(uint64, uint64) { evictions++ })
+	defer g.Close()
+	check := func(phase string) {
+		t.Helper()
+		require.Equal(t, g.cur.Load().lru.Len(), g.Len(), "entry counter drifted after %s", phase)
+	}
+	// growLRU hashes keys with the identity function, so spread the shard-selection
+	// bits the way its real maphash/keccak-derived keys do.
+	key := func(i uint64) uint64 { return i * 0x9E3779B97F4A7C15 }
+
+	for i := range uint64(500) {
+		g.Put(key(i), i)
+	}
+	check("adds")
+	require.Equal(t, 500, g.Len(), "500 distinct keys must fit below the 1024-slot start capacity")
+
+	for i := range uint64(100) {
+		g.Remove(key(i))
+	}
+	check("removes")
+	require.Equal(t, 100, evictions, "the caller's OnEvict must still fire for each removal")
+
+	before := g.cur.Load()
+	for i := uint64(500); i < 4000; i++ {
+		g.Put(key(i), i)
+	}
+	require.NotEqual(t, before, g.cur.Load(), "grow did not happen")
+	check("grow")
+
+	g.Purge()
+	require.Equal(t, 0, g.Len())
+	check("purge")
+}
+
+// A grow can copy a key into the new generation while a same-key refresh is in
+// flight. Put resolves g.cur once so its removal and its store land on the same
+// generation; split across the swap, the store would replace the copy in place
+// without firing OnEvict and double-count it.
+func TestGrowLRU_GrowRacePutDoesNotDoubleCount(t *testing.T) {
+	g := newGrowLRU[uint64](8*datasize.MB, 16, nil)
+	defer g.Close()
+
+	h := uint64(7)
+	g.Put(h, 1)
+	gen1 := g.cur.Load()
+
+	// Build the next generation exactly like maybeGrow's copy loop, and
+	// publish it while h is still present in gen1.
+	newCap := g.curCap.Load() * genericCacheGrowFactor
+	gen2 := g.newShards(newCap)
+	for _, k := range gen1.lru.Keys() {
+		if v, ok := gen1.lru.Get(k); ok {
+			gen2.add(k, v)
+		}
+	}
+	g.cur.Store(gen2)
+	g.curCap.Store(newCap)
+
+	g.Put(h, 2)
+
+	require.Equal(t, gen2.lru.Len(), g.Len(),
+		"counter must not double-count a grow-copied key replaced in place")
+	got, ok := g.Get(h)
+	require.True(t, ok)
+	require.Equal(t, uint64(2), got, "the refreshed value must be the one served")
+
+	// A refresh that resolved the retired generation must leave the live count alone.
+	live := g.Len()
+	gen1.lru.Remove(h)
+	gen1.add(h, 3)
+	require.Equal(t, live, g.Len(), "a write lost in the retired generation must not move the live counter")
+	require.Equal(t, gen1.lru.Len(), gen1.len(), "the retired generation's own counter must stay exact")
+}
+
+// Each layer's counter must equal its LRU's real length while all three are
+// driven through putContentLocked's displacing path. The fixture repeats keys
+// and keeps unwinding them so that path runs, with budget left to grow into.
+func TestCodeCache_GrowLRULenCounterUnderConcurrency(t *testing.T) {
+	// 64MB over avgCodeEntryBytes puts the two code layers' ceiling well above
+	// genericCacheStartCapacity; at 8MB it lands below and they never grow.
+	cc := closeOnCleanup(t, NewCodeCache(64*datasize.MB, 8*datasize.MB))
+	// Floor 0, so every later Unwind turns the whole resident set stale on its
+	// epoch bump alone.
+	cc.Unwind(0)
+
+	const workers = 8
+	const perWorker = 3000
+	const keySpace = 4096
+
+	var unwinder sync.WaitGroup
+	done := make(chan struct{})
+	unwinder.Go(func() {
+		for {
+			select {
+			case <-done:
+				return
+			default:
+			}
+			cc.Unwind(0)
+			runtime.Gosched()
+		}
+	})
+
+	var wg sync.WaitGroup
+	for w := range workers {
+		wg.Go(func() {
+			addr := make([]byte, 20)
+			code := make([]byte, 40)
+			codeHash := make([]byte, 32)
+			for i := range perWorker {
+				k := uint64(w*perWorker+i) % keySpace
+				binary.BigEndian.PutUint64(addr[12:], k)
+				binary.BigEndian.PutUint64(code, k)
+				binary.BigEndian.PutUint64(codeHash, k)
+				cc.PutWithCodeHash(addr, code, codeHash, uint64(i))
+			}
+		})
+	}
+	wg.Wait()
+	close(done)
+	unwinder.Wait()
+
+	for _, layer := range []struct {
+		name string
+		lru  *growLRU[codeEntry]
+	}{
+		{"hashToCode", cc.hashToCode},
+		{"codeHashToCode", cc.codeHashToCode},
+	} {
+		require.Equal(t, layer.lru.cur.Load().lru.Len(), layer.lru.Len(), "%s counter drifted", layer.name)
+		require.Greater(t, layer.lru.curCap.Load(), layer.lru.startCap, "%s never grew", layer.name)
+	}
+	require.Equal(t, cc.codeSizeByCodeHash.cur.Load().lru.Len(), cc.codeSizeByCodeHash.Len(),
+		"codeSizeByCodeHash counter drifted")
+	require.Greater(t, cc.codeSizeByCodeHash.curCap.Load(), cc.codeSizeByCodeHash.startCap,
+		"codeSizeByCodeHash never grew")
+}
+
+// Parallel adds while the LRU is still below its ceiling — the window where
+// every add runs the grow check.
+func BenchmarkGrowLRUParallelPutGrow(b *testing.B) {
+	g := newGrowLRU[uint64](256*datasize.MB, 48, nil)
+	defer g.Close()
+	var seq atomic.Uint64
+	b.ResetTimer()
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			n := seq.Add(1) * 0x9E3779B97F4A7C15
+			g.Put(n, n)
+		}
+	})
+}
+
+// CodeCache's read-path stale drops hold no put stripe, so they interleave
+// freely with a striped refresh while the LRU grows. The counter must still
+// equal the real length: both grow gates read it, so an under-count wedges
+// growth for good.
+func TestGrowLRU_CountExactUnderStripedRefreshAndUnstripedRemove(t *testing.T) {
+	g := newGrowLRU[uint64](8*datasize.MB, 16, nil)
+	defer g.Close()
+
+	const keySpace = 4096
+	key := func(i uint64) uint64 { return (i % keySpace) * 0x9E3779B97F4A7C15 }
+
+	var stripes [256]sync.Mutex
+	var wg sync.WaitGroup
+	for w := range 8 {
+		wg.Go(func() {
+			for i := range uint64(20000) {
+				h := key(uint64(w)*20000 + i)
+				stripe := &stripes[uint8(h)]
+				stripe.Lock()
+				g.Put(h, i)
+				stripe.Unlock()
+			}
+		})
+	}
+	wg.Go(func() {
+		for i := range uint64(60000) {
+			g.Remove(key(i))
+		}
+	})
+	wg.Wait()
+
+	gen := g.cur.Load()
+	require.Equal(t, gen.lru.Len(), gen.len(), "live generation's counter drifted from its real length")
+	require.NotEqual(t, g.startCap, g.curCap.Load(), "the LRU never grew, so the growth gates were not exercised")
+}