From 222f5aa887eac1fa0a8388ecdf6e6ce4fdf159b8 Mon Sep 17 00:00:00 2001 From: awskii Date: Tue, 7 Jul 2026 16:54:58 +0700 Subject: [PATCH 1/9] execution/commitment: defer parallel trie selection to EnableParaTrieDB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With COMMITMENT_PARALLEL set globally, every SharedDomains selected the parallel trie, but only DB-backed consumers (exec, builder, squeeze, backtester) wire the per-worker TrieContextFactory it needs — integrity checks, RPC-created domains, and test harnesses failed with 'ParallelPatriciaHashed.Process requires a TrieContextFactory'. The context now starts on the sequential trie and upgrades to the selected parallel/streaming variant when EnableParaTrieDB provides the DB. SeekCommitment may restore state before the DB is wired, so the upgrade adopts the already-restored trie as the parallel template instead of re-encoding (SetState re-reads a sole-account root through the not-yet-installed context). Touching before the upgrade panics: keys collected on the sequential buffer would be dropped. (cherry picked from commit 1436a650f18f967020261d8a654a7ad24b5e48a4) --- .../commitmentdb/commitment_context.go | 41 ++++++++++++++++++- .../commitment/parallel_patricia_hashed.go | 7 ++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index d31f16ac163..18bad0f186d 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -74,6 +74,11 @@ type SharedDomainsCommitmentContext struct { deferCommitmentUpdates bool // pendingUpdate stores a single deferred branch update to be flushed at the next ComputeCommitment call. pendingUpdate *commitment.PendingCommitmentUpdate + + // pendingVariant holds a parallel/streaming trie selection that waits for + // EnableParaTrieDB: those variants need the DB-backed TrieContextFactory. + pendingVariant commitment.TrieVariant + pendingCfg commitment.TrieConfig } // SetStateReader can be used to set a custom state reader (otherwise the default one is set in SharedDomainsCommitmentContext.trieContext). @@ -89,6 +94,30 @@ func (sdc *SharedDomainsCommitmentContext) StateReader() StateReader { func (sdc *SharedDomainsCommitmentContext) EnableParaTrieDB(db kv.TemporalRoDB) { sdc.paraTrieDB = db + if sdc.pendingVariant == "" { + return + } + if sdc.updates.Size() != 0 { + panic("EnableParaTrieDB after touches: keys collected on the sequential buffer would be dropped") + } + prev, ok := sdc.patriciaTrie.(*commitment.HexPatriciaHashed) + if !ok { + panic("pending trie upgrade expects the sequential trie") + } + cfg := sdc.pendingCfg + cfg.Variant = sdc.pendingVariant + sdc.updates.Close() + sdc.patriciaTrie, sdc.updates = commitment.InitializeTrieAndUpdates(commitment.ModeDirect, sdc.tmpDir, cfg) + if ppht, ok := sdc.patriciaTrie.(*commitment.ParallelPatriciaHashed); ok { + // State may already be restored (SeekCommitment can run before the DB + // is wired); adopting the trie carries it over losslessly. + ppht.AdoptRootTrie(prev) + } + sdc.variant = sdc.pendingVariant + sdc.pendingVariant = "" + if sdc.traceW != nil { + sdc.patriciaTrie.SetTraceWriter(sdc.traceW) + } } // EnableTrieWarmup enables parallel warmup of MDBX page cache during commitment. @@ -196,12 +225,22 @@ func NewSharedDomainsCommitmentContext(sd sd, mode commitment.Mode, tmpDir strin ctx := &SharedDomainsCommitmentContext{ sharedDomains: sd, tmpDir: tmpDir, - variant: variant, + variant: commitment.VariantHexPatriciaTrie, warmupBase: commitment.WarmupConfig{ Enabled: cfg.EnableTrieWarmup, NumWorkers: cfg.WarmupNumWorkersOrDefault(), }, } + // The parallel and streaming tries need a per-worker TrieContextFactory that + // only DB-backed consumers can provide (via EnableParaTrieDB). Start on the + // sequential trie and upgrade when the DB arrives, so context holders that + // never wire one (RPC, integrity, tests) keep working under a global variant + // selection. + if variant == commitment.VariantParallelHexPatricia || variant == commitment.VariantStreamingHexPatricia { + ctx.pendingVariant = variant + cfg.Variant = commitment.VariantHexPatriciaTrie + ctx.pendingCfg = cfg + } ctx.patriciaTrie, ctx.updates = commitment.InitializeTrieAndUpdates(mode, tmpDir, cfg) return ctx } diff --git a/execution/commitment/parallel_patricia_hashed.go b/execution/commitment/parallel_patricia_hashed.go index e5283a10ce5..ee79b664642 100644 --- a/execution/commitment/parallel_patricia_hashed.go +++ b/execution/commitment/parallel_patricia_hashed.go @@ -100,6 +100,13 @@ func (p *ParallelPatriciaHashed) TakeDeferredUpdates() []*DeferredBranchUpdate { } // RootTrie exposes the configuration template only; it must not be used as live root state. +// AdoptRootTrie replaces the template trie with one that already carries state +// (e.g. restored before the variant upgrade), the same slot state restore +// targets; the previous trie must no longer be used. +func (p *ParallelPatriciaHashed) AdoptRootTrie(root *HexPatriciaHashed) { + p.template = root +} + func (p *ParallelPatriciaHashed) RootTrie() *HexPatriciaHashed { return p.template } From 11b22b12a17560c437a64c0f673c9705e2dc67ad Mon Sep 17 00:00:00 2001 From: awskii Date: Wed, 8 Jul 2026 15:26:47 +0700 Subject: [PATCH 2/9] db/state, db/kv, execution/commitment: pin file generation for parallel-commitment worker reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parallel-commitment workers each opened a fresh BeginTemporalRo, pinning whatever aggregator visible-file generation was current — possibly newer than the main commitment tx. A deleted/reincarnated account then read AccountsDomain=empty (from the in-memory overlay) but CodeDomain=stale (from the worker's newer file view, since the code delete leaves no mem tombstone when prevVal is nil), failing the ERIGON_ASSERT code-hash consistency check and wedging the node. Serial commitment reads through one tx and stayed consistent, so this only surfaced once CI began exercising the parallel-commitment axis. Add a files-pin API (AggregatorRoTx.Pin -> AggregatorFilesPin, kv.TemporalFilesPin): ComputeCommitment pins the main tx's file generation and opens worker read txns from it, so all concurrent workers observe the snapshot the in-memory overlay was built against. Forwarded through the block-overlay wrapper for the builder path; falls back (with a warning) for backends that can't pin files. (cherry picked from commit a8cb055c2904eb097dd10c7bad9050326e3465b2) --- db/kv/kv_interface.go | 12 +++ db/kv/membatchwithdb/memory_mutation.go | 13 ++++ db/kv/temporal/kv_temporal.go | 36 +++++++++ db/state/aggregator.go | 42 +++++++++- db/state/aggregator_visible_from_test.go | 76 +++++++++++++++++++ .../commitmentdb/commitment_context.go | 49 ++++++++++-- 6 files changed, 221 insertions(+), 7 deletions(-) create mode 100644 db/state/aggregator_visible_from_test.go diff --git a/db/kv/kv_interface.go b/db/kv/kv_interface.go index 3aa7daeb5f8..78882dbf7d5 100644 --- a/db/kv/kv_interface.go +++ b/db/kv/kv_interface.go @@ -475,6 +475,18 @@ type TemporalTx interface { AggTx() any } +// TemporalFilesPin holds a consistent aggregator file snapshot so that multiple +// read txns opened from it (BeginTemporalRo) all observe the same file +// generation, even after newer generations are published. Concurrent readers +// spawned from one commitment tx use this to avoid reading a domain from a file +// generation inconsistent with the in-memory overlay that tx was built against. +// Release with Close. A temporal tx exposes it via an optional `Pin() +// TemporalFilesPin` method (type-asserted; not all backends can pin files). +type TemporalFilesPin interface { + BeginTemporalRo(ctx context.Context) (TemporalTx, error) + Close() +} + // TemporalDebugTx - set of slow low-level funcs for debug purposes type TemporalDebugTx interface { RangeLatest(domain Domain, from, to []byte, limit int) (stream.KV, error) diff --git a/db/kv/membatchwithdb/memory_mutation.go b/db/kv/membatchwithdb/memory_mutation.go index 88a69f9cf39..e220fa60094 100644 --- a/db/kv/membatchwithdb/memory_mutation.go +++ b/db/kv/membatchwithdb/memory_mutation.go @@ -115,6 +115,19 @@ func (m *MemoryMutation) UnderlyingTx() kv.TemporalTx { return m.db } +// Pin forwards the files-pin capability to the underlying tx so that parallel +// commitment workers reading through an overlay view still pin the same file +// generation as the main read — domain (Accounts/Code/Storage) reads fall +// through the overlay to the underlying tx, so pinning it keeps worker reads +// consistent. Returns nil when the underlying tx can't pin files, letting the +// caller fall back to an independent snapshot. +func (m *MemoryMutation) Pin() kv.TemporalFilesPin { + if p, ok := m.db.(interface{ Pin() kv.TemporalFilesPin }); ok { + return p.Pin() + } + return nil +} + func (m *MemoryMutation) UpdateTxn(tx kv.TemporalTx) { m.mu.Lock() defer m.mu.Unlock() diff --git a/db/kv/temporal/kv_temporal.go b/db/kv/temporal/kv_temporal.go index 19f8657d069..a2306cc21fc 100644 --- a/db/kv/temporal/kv_temporal.go +++ b/db/kv/temporal/kv_temporal.go @@ -97,6 +97,42 @@ func (db *DB) BeginTemporalRo(ctx context.Context) (kv.TemporalTx, error) { return tx, nil } + +// temporalFilesPin implements kv.TemporalFilesPin: it holds a consistent +// aggregator file snapshot and opens read txns bound to it. +type temporalFilesPin struct { + db *DB + agg *state.AggregatorFilesPin +} + +// Pin holds this tx's aggregator file snapshot so consistent read txns can be +// opened from it (BeginTemporalRo) even after newer file generations are +// published — used by parallel-commitment workers so they never read a domain +// from a generation inconsistent with the in-memory overlay this tx was built +// against. The pin is independent of this tx's lifetime; release it with Close. +func (tx *tx) Pin() kv.TemporalFilesPin { + return &temporalFilesPin{db: tx.db, agg: tx.aggtx.Pin()} +} + +func (p *temporalFilesPin) BeginTemporalRo(ctx context.Context) (kv.TemporalTx, error) { + kvTx, err := p.db.RwDB.BeginRo(ctx) //nolint:gocritic + if err != nil { + return nil, err + } + tx := &Tx{Tx: kvTx, tx: tx{db: p.db, ctx: ctx}} + tx.aggtx = p.agg.BeginFilesRo() + + if len(p.db.forkaggs) > 0 { + tx.forkaggs = make([]*state.ForkableAggTemporalTx, len(p.db.forkaggs)) + for i, forkagg := range p.db.forkaggs { + tx.forkaggs[i] = forkagg.BeginTemporalTx() + } + } + return tx, nil +} + +func (p *temporalFilesPin) Close() { p.agg.Close() } + func (db *DB) ViewTemporal(ctx context.Context, f func(tx kv.TemporalTx) error) error { tx, err := db.BeginTemporalRo(ctx) if err != nil { diff --git a/db/state/aggregator.go b/db/state/aggregator.go index 91ae7bd5fe0..c890c4d1557 100644 --- a/db/state/aggregator.go +++ b/db/state/aggregator.go @@ -2388,7 +2388,47 @@ func closeAndRemoveFiles(files []*FilesItem) { } func (a *Aggregator) BeginFilesRo() *AggregatorRoTx { - v := a.acquireVisibleFiles() + return a.beginFilesRoOn(a.acquireVisibleFiles()) +} + +// AggregatorFilesPin is a refcounted hold on one visible-file generation. +// AggregatorRoTx values opened from it via BeginFilesRo all observe that same +// snapshot, regardless of newer generations published meanwhile — so concurrent +// readers derived from a single commitment tx never diverge across file +// generations. It outlives the tx it was pinned from; release with Close. +type AggregatorFilesPin struct { + a *Aggregator + v *aggregatorVisible +} + +// Pin takes an independent refcount on this tx's visible-file generation so +// consistent read txns can be spawned from it for the pin's lifetime. The +// generation cannot be reclaimed between the load and the bump because this tx +// already holds a pin on it. +func (at *AggregatorRoTx) Pin() *AggregatorFilesPin { + at.visible.refcnt.Add(1) + return &AggregatorFilesPin{a: at.a, v: at.visible} +} + +// BeginFilesRo opens a fresh AggregatorRoTx (its own cursors) pinned to the +// pin's generation. +func (p *AggregatorFilesPin) BeginFilesRo() *AggregatorRoTx { + p.v.refcnt.Add(1) + return p.a.beginFilesRoOn(p.v) +} + +// Close releases the pin's refcount on the generation. +func (p *AggregatorFilesPin) Close() { + if p.a != nil { + p.a.releaseVisibleFiles(p.v) + p.a, p.v = nil, nil + } +} + +// beginFilesRoOn builds an AggregatorRoTx with fresh per-domain/per-index cursors +// over the already-pinned visible generation v (caller owns the refcnt on v, +// released by AggregatorRoTx.Close). +func (a *Aggregator) beginFilesRoOn(v *aggregatorVisible) *AggregatorRoTx { ac := &AggregatorRoTx{ a: a, visible: v, diff --git a/db/state/aggregator_visible_from_test.go b/db/state/aggregator_visible_from_test.go new file mode 100644 index 00000000000..aca36230d10 --- /dev/null +++ b/db/state/aggregator_visible_from_test.go @@ -0,0 +1,76 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package state + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// A worker tx opened via BeginFilesRoFrom must pin the SAME visible-file +// generation as its source tx, even after a newer generation is published — so +// parallel-commitment workers spawned from one commitment tx never read a domain +// from a file generation inconsistent with the in-memory overlay that tx was +// built against (the torn Accounts-vs-Code read that fails the code-hash assert). +// A plain BeginFilesRo, by contrast, pins whatever generation is current when it +// runs, which is the source of that inconsistency. +func TestFilesPin_PinsSourceGeneration(t *testing.T) { + stepSize := uint64(10) + _, agg := testDbAndAggregatorv3(t, stepSize) + + gen := func(ranges []testFileRange) { + t.Helper() + generateAccountsFile(t, agg.Dirs(), ranges) + generateCodeFile(t, agg.Dirs(), ranges) + generateStorageFile(t, agg.Dirs(), ranges) + generateCommitmentFile(t, agg.Dirs(), ranges) + require.NoError(t, agg.OpenFolder()) + } + + gen([]testFileRange{{0, 1}}) + src := agg.BeginFilesRo() + defer src.Close() + genV1 := src.visible + require.Same(t, agg.visible.Load(), genV1, "src pins the current generation") + + // Publish a newer generation while src stays open. + gen([]testFileRange{{0, 1}, {1, 2}}) + genV2 := agg.visible.Load() + require.NotSame(t, genV1, genV2, "OpenFolder must publish a newer visible generation") + + // The fix: pin the source generation, then txns opened from the pin keep it, + // not the newer one. + pin := src.Pin() + defer pin.Close() + worker := pin.BeginFilesRo() + require.Same(t, genV1, worker.visible, "pin.BeginFilesRo must pin the source generation") + worker.Close() + + // Contrast — a plain BeginFilesRo pins the current (newer) generation. That is + // the divergence the pin removes for parallel-commitment workers. + fresh := agg.BeginFilesRo() + require.Same(t, genV2, fresh.visible, "BeginFilesRo pins the current (newer) generation") + fresh.Close() + + // Refcnt: a txn from the pin takes a pin on the source generation that Close releases. + before := genV1.refcnt.Load() + w2 := pin.BeginFilesRo() + require.Equal(t, before+1, genV1.refcnt.Load(), "pin.BeginFilesRo increments the source generation refcnt") + w2.Close() + require.Equal(t, before, genV1.refcnt.Load(), "Close releases the pin") +} diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 18bad0f186d..2ed33e09d55 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -541,15 +541,33 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context warmupConfig = sdc.warmupBase warmupConfig.MaxDepth = commitment.WarmupMaxDepth warmupConfig.LogPrefix = logPrefix + // Pin the file generation this commitment (and its in-memory overlay) is + // bound to, so parallel/warmup worker reads stay consistent with it — a + // worker opening a fresh snapshot could pin a newer generation and read a + // domain (e.g. code) inconsistent with the account overlay. + var workerPin kv.TemporalFilesPin + if p, ok := tx.(filesPinner); ok { + if wp := p.Pin(); wp != nil { + workerPin = wp + defer workerPin.Close() + } + } switch trie := sdc.patriciaTrie.(type) { case *commitment.ParallelPatriciaHashed: + // A parallel commitment without a pinned snapshot means the backing tx + // can't pin its file generation, so concurrent worker reads may diverge + // from the main read across file generations (the torn Accounts/Code + // read). Expected only for non-pinnable backends (some tests/mocks). + if workerPin == nil { + log.Warn("[commitment] parallel commitment without a pinned file snapshot; worker reads not generation-consistent", "logPrefix", logPrefix) + } // Each worker writes its branch updates through a private collector // so concurrent PutBranch calls never race; collectors are drained // after Process and merged into the main writer below. - warmupConfig.CtxFactory, drainCollectors = sdc.concurrentTrieContextFactory(ctx, sdc.paraTrieDB, txNum) + warmupConfig.CtxFactory, drainCollectors = sdc.concurrentTrieContextFactory(ctx, sdc.paraTrieDB, workerPin, txNum) trie.SetTrieContextFactory(warmupConfig.CtxFactory) default: - warmupConfig.CtxFactory = sdc.trieContextFactory(ctx, sdc.paraTrieDB, txNum) + warmupConfig.CtxFactory = sdc.trieContextFactory(ctx, sdc.paraTrieDB, workerPin, txNum) } } @@ -629,11 +647,30 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context return rootHash, err } -func (sdc *SharedDomainsCommitmentContext) trieContextFactory(ctx context.Context, db kv.TemporalRoDB, txNum uint64) commitment.TrieContextFactory { +// filesPinner is the optional capability of a temporal tx to pin its visible +// file generation (see kv.TemporalFilesPin). +type filesPinner interface { + Pin() kv.TemporalFilesPin +} + +// beginWorkerRo opens a per-worker read tx for parallel/warmup commitment reads. +// When a files pin is available it opens the tx bound to that snapshot, so a +// worker never reads a domain from a newer file generation than the in-memory +// overlay was built against — the torn Accounts-vs-Code read that fails the +// code-hash assert. Without a pin (a backend that can't pin files) it falls back +// to an independent snapshot. +func beginWorkerRo(ctx context.Context, db kv.TemporalRoDB, pin kv.TemporalFilesPin) (kv.TemporalTx, error) { + if pin != nil { + return pin.BeginTemporalRo(ctx) + } + return db.BeginTemporalRo(ctx) +} + +func (sdc *SharedDomainsCommitmentContext) trieContextFactory(ctx context.Context, db kv.TemporalRoDB, pin kv.TemporalFilesPin, txNum uint64) commitment.TrieContextFactory { // avoid races like this stepSize := sdc.sharedDomains.StepSize() return func() (commitment.PatriciaContext, func()) { - roTx, err := db.BeginTemporalRo(ctx) //nolint:gocritic + roTx, err := beginWorkerRo(ctx, db, pin) //nolint:gocritic if err != nil { return &errorTrieContext{err: err}, func() {} } @@ -668,13 +705,13 @@ func (sdc *SharedDomainsCommitmentContext) trieContextFactory(ctx context.Contex // concurrentTrieContextFactory is like trieContextFactory but also creates a per-goroutine // etl.Collector for each context so that PutBranch writes are isolated (no shared writer race). // Returns the factory and a drain function that collects all created collectors. -func (sdc *SharedDomainsCommitmentContext) concurrentTrieContextFactory(ctx context.Context, db kv.TemporalRoDB, txNum uint64) (commitment.TrieContextFactory, func() []*etl.Collector) { +func (sdc *SharedDomainsCommitmentContext) concurrentTrieContextFactory(ctx context.Context, db kv.TemporalRoDB, pin kv.TemporalFilesPin, txNum uint64) (commitment.TrieContextFactory, func() []*etl.Collector) { stepSize := sdc.sharedDomains.StepSize() var mu sync.Mutex var collectors []*etl.Collector factory := func() (commitment.PatriciaContext, func()) { - roTx, err := db.BeginTemporalRo(ctx) //nolint:gocritic + roTx, err := beginWorkerRo(ctx, db, pin) //nolint:gocritic if err != nil { return &errorTrieContext{err: err}, func() {} } From b1f931f9b50fe2378caf48a492103a7c121c5a05 Mon Sep 17 00:00:00 2001 From: awskii Date: Wed, 8 Jul 2026 15:55:06 +0700 Subject: [PATCH 3/9] db/kv/temporal: drop forkaggs from files-pin worker tx (removed on main) The pin's worker tx copied BeginTemporalRo's forkaggs setup, which main removed; the branch+main merge build then failed on the now-undefined forkaggs. Commitment workers read only state domains via aggtx, so the worker tx needs just the pinned file snapshot. (cherry picked from commit bd0dbb0763202d75823d5c1eff26cc85b82b01bf) --- db/kv/temporal/kv_temporal.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/db/kv/temporal/kv_temporal.go b/db/kv/temporal/kv_temporal.go index a2306cc21fc..df9442cfe5f 100644 --- a/db/kv/temporal/kv_temporal.go +++ b/db/kv/temporal/kv_temporal.go @@ -119,15 +119,10 @@ func (p *temporalFilesPin) BeginTemporalRo(ctx context.Context) (kv.TemporalTx, if err != nil { return nil, err } + // Commitment workers read only state domains through aggtx, never forkable + // data, so the worker tx needs the pinned file snapshot and nothing else. tx := &Tx{Tx: kvTx, tx: tx{db: p.db, ctx: ctx}} tx.aggtx = p.agg.BeginFilesRo() - - if len(p.db.forkaggs) > 0 { - tx.forkaggs = make([]*state.ForkableAggTemporalTx, len(p.db.forkaggs)) - for i, forkagg := range p.db.forkaggs { - tx.forkaggs[i] = forkagg.BeginTemporalTx() - } - } return tx, nil } From 9b9612bf98a6050551120eae1a98039a53e91c64 Mon Sep 17 00:00:00 2001 From: awskii Date: Wed, 8 Jul 2026 16:46:34 +0700 Subject: [PATCH 4/9] execution/commitment: pin only the parallel-trie fold factory, not serial warmup The pin was also applied to the warmup factory, which serves serial/streaming commitment's page-cache warmup. Warmup does not compute the root, and pinning it regressed serial commitment to a wrong trie root on mainnet. Restrict the pin to the ParallelPatriciaHashed fold factory (which computes the root and had the torn Accounts/Code read); serial warmup keeps its independent snapshot, so the serial path is behaviorally identical to before the pin change. (cherry picked from commit 995f74cd6fe2707241a32d06635bff344518ffbb) --- .../commitmentdb/commitment_context.go | 35 +++++++++---------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 2ed33e09d55..dc209e500e0 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -541,23 +541,20 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context warmupConfig = sdc.warmupBase warmupConfig.MaxDepth = commitment.WarmupMaxDepth warmupConfig.LogPrefix = logPrefix - // Pin the file generation this commitment (and its in-memory overlay) is - // bound to, so parallel/warmup worker reads stay consistent with it — a - // worker opening a fresh snapshot could pin a newer generation and read a - // domain (e.g. code) inconsistent with the account overlay. - var workerPin kv.TemporalFilesPin - if p, ok := tx.(filesPinner); ok { - if wp := p.Pin(); wp != nil { - workerPin = wp - defer workerPin.Close() - } - } switch trie := sdc.patriciaTrie.(type) { case *commitment.ParallelPatriciaHashed: - // A parallel commitment without a pinned snapshot means the backing tx - // can't pin its file generation, so concurrent worker reads may diverge - // from the main read across file generations (the torn Accounts/Code - // read). Expected only for non-pinnable backends (some tests/mocks). + // The parallel fold workers compute the root, so they must read the same + // file generation the in-memory overlay was built against: pin the main + // tx's generation and open worker txns from it. Otherwise a worker could + // pin a newer generation and read a domain (e.g. code) inconsistent with + // the account overlay (the torn Accounts/Code read). + var workerPin kv.TemporalFilesPin + if p, ok := tx.(filesPinner); ok { + if wp := p.Pin(); wp != nil { + workerPin = wp + defer workerPin.Close() + } + } if workerPin == nil { log.Warn("[commitment] parallel commitment without a pinned file snapshot; worker reads not generation-consistent", "logPrefix", logPrefix) } @@ -567,7 +564,9 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context warmupConfig.CtxFactory, drainCollectors = sdc.concurrentTrieContextFactory(ctx, sdc.paraTrieDB, workerPin, txNum) trie.SetTrieContextFactory(warmupConfig.CtxFactory) default: - warmupConfig.CtxFactory = sdc.trieContextFactory(ctx, sdc.paraTrieDB, workerPin, txNum) + // Serial/streaming: this factory only serves page-cache warmup, which + // does not compute the root, so its reads need no generation pin. + warmupConfig.CtxFactory = sdc.trieContextFactory(ctx, sdc.paraTrieDB, txNum) } } @@ -666,11 +665,11 @@ func beginWorkerRo(ctx context.Context, db kv.TemporalRoDB, pin kv.TemporalFiles return db.BeginTemporalRo(ctx) } -func (sdc *SharedDomainsCommitmentContext) trieContextFactory(ctx context.Context, db kv.TemporalRoDB, pin kv.TemporalFilesPin, txNum uint64) commitment.TrieContextFactory { +func (sdc *SharedDomainsCommitmentContext) trieContextFactory(ctx context.Context, db kv.TemporalRoDB, txNum uint64) commitment.TrieContextFactory { // avoid races like this stepSize := sdc.sharedDomains.StepSize() return func() (commitment.PatriciaContext, func()) { - roTx, err := beginWorkerRo(ctx, db, pin) //nolint:gocritic + roTx, err := db.BeginTemporalRo(ctx) //nolint:gocritic if err != nil { return &errorTrieContext{err: err}, func() {} } From b16d560ac952ed6e521fdf7933cab9f40a14db68 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Thu, 9 Jul 2026 19:00:09 +0700 Subject: [PATCH 5/9] ci, db/state: matrix-test serial vs parallel commitment across the test workflows Repurpose the (now-redundant) serial/parallel exec-mode CI matrix to exercise serial vs parallel commitment instead: the exec_mode axis is renamed commitment_mode, ERIGON_EXEC3_PARALLEL assignments are dropped (exec is parallel in both legs by default), and ERIGON_COMMITMENT_PARALLEL is driven from the axis. state_schema.go reads COMMITMENT_PARALLEL via dbg.EnvBool; the integration BoolVar defaults to the env-derived value. EEST spec/devnet shards select the mode via their -parallel suffix. Squash of the three CI commits from #22141 (4d1d24ec, f86ceab9, c923faae), reconciled with main's consume-enginex migration (#22344) in qa-stage-exec.yml and test-hive-eest.yml, and rebased onto the extracted parallel-commitment fixes (#22360) so the CI axis no longer carries the code changes it exercises. --- .../qa-rpc-integration-tests-latest.yml | 20 +++---- .../qa-rpc-performance-comparison-tests.yml | 26 ++++----- .../workflows/qa-rpc-performance-tests.yml | 26 ++++----- .github/workflows/qa-stage-exec.yml | 40 +++++++------- .../workflows/qa-txpool-performance-test.yml | 30 +++++------ .github/workflows/test-all-erigon-race.yml | 27 +++++----- .github/workflows/test-all-erigon.yml | 29 +++++----- .github/workflows/test-bench.yml | 17 +++--- .github/workflows/test-hive-eest.yml | 51 +++++++++--------- .github/workflows/test-hive.yml | 53 ++++++++++--------- .github/workflows/test-kurtosis-assertoor.yml | 45 ++++++++-------- cmd/integration/commands/flags.go | 4 +- db/state/statecfg/state_schema.go | 5 +- tools/eest-spec-shards.yml | 34 ++++++------ tools/run-eest-spec-test.sh | 37 ++++++------- 15 files changed, 227 insertions(+), 217 deletions(-) diff --git a/.github/workflows/qa-rpc-integration-tests-latest.yml b/.github/workflows/qa-rpc-integration-tests-latest.yml index da22e7a2867..3b758c43b19 100644 --- a/.github/workflows/qa-rpc-integration-tests-latest.yml +++ b/.github/workflows/qa-rpc-integration-tests-latest.yml @@ -23,7 +23,7 @@ on: jobs: mainnet-rpc-integ-tests-latest: - name: mainnet-rpc-integ-tests-latest (${{ matrix.exec_mode }}) + name: mainnet-rpc-integ-tests-latest (${{ matrix.commitment_mode }}) concurrency: group: >- ${{ @@ -42,7 +42,7 @@ jobs: # just one after the other (~2× wall-clock). max-parallel: 1 matrix: - exec_mode: + commitment_mode: - serial - parallel env: @@ -53,10 +53,10 @@ jobs: ERIGON_ASSERT: true RPC_PAST_TEST_DIR: /opt/rpc-past-tests CHAIN: mainnet - # Toggle dbg.Exec3Parallel from CI without code changes. envLookup - # in common/dbg/dbg_env.go auto-prepends ERIGON_, so this maps to - # the EXEC3_PARALLEL flag declared in common/dbg/experiments.go. - ERIGON_EXEC3_PARALLEL: ${{ matrix.exec_mode == 'parallel' && 'true' || 'false' }} + # Toggle statecfg.ExperimentalParallelCommitment from CI without code + # changes. envLookup in common/dbg/dbg_env.go auto-prepends ERIGON_, so + # this maps to the COMMITMENT_PARALLEL flag read in db/state/statecfg. + ERIGON_COMMITMENT_PARALLEL: ${{ matrix.commitment_mode == 'parallel' && 'true' || 'false' }} steps: - name: Check out repository @@ -123,7 +123,7 @@ jobs: if: failure() && steps.preparing_step.outcome == 'failure' uses: actions/upload-artifact@v7 with: - name: preparing-step-logs-${{ matrix.exec_mode }} + name: preparing-step-logs-${{ matrix.commitment_mode }} path: ${{ env.ERIGON_REFERENCE_DATA_DIR }}/logs/ - name: Pause the Erigon instance dedicated to db maintenance @@ -180,7 +180,7 @@ jobs: if: failure() && steps.pre_test_step.outcome == 'failure' uses: actions/upload-artifact@v7 with: - name: pre-test-logs-${{ matrix.exec_mode }} + name: pre-test-logs-${{ matrix.commitment_mode }} path: ${{ env.ERIGON_TESTBED_DATA_DIR }}/logs/ - name: Run RPC Integration Tests @@ -260,7 +260,7 @@ jobs: if: always() && steps.test_step.outputs.test_executed == 'true' uses: actions/upload-artifact@v7 with: - name: test-results-${{ matrix.exec_mode }} + name: test-results-${{ matrix.commitment_mode }} path: | ${{ env.TEST_RESULT_DIR }} ${{ env.ERIGON_TESTBED_DATA_DIR }}/logs/ @@ -280,7 +280,7 @@ jobs: --repo erigon \ --commit $(git rev-parse HEAD) \ --branch ${{ github.ref_name }} \ - --test_name rpc-integration-tests-latest${{ matrix.exec_mode == 'parallel' && '-parallel' || '' }} \ + --test_name rpc-integration-tests-latest${{ matrix.commitment_mode == 'parallel' && '-parallel' || '' }} \ --chain $CHAIN \ --runner ${{ runner.name }} \ --db_version $db_version \ diff --git a/.github/workflows/qa-rpc-performance-comparison-tests.yml b/.github/workflows/qa-rpc-performance-comparison-tests.yml index 6127920cde0..e07970cd891 100644 --- a/.github/workflows/qa-rpc-performance-comparison-tests.yml +++ b/.github/workflows/qa-rpc-performance-comparison-tests.yml @@ -8,8 +8,8 @@ on: type: boolean required: false default: false - exec_mode: - description: 'Erigon execution mode ("default" leaves ERIGON_EXEC3_PARALLEL unset)' + commitment_mode: + description: 'Erigon commitment mode ("default" leaves ERIGON_COMMITMENT_PARALLEL unset)' type: choice required: false default: default @@ -194,22 +194,22 @@ jobs: id: erigon_running_step working-directory: ${{ github.workspace }}/build/bin env: - EXEC_MODE: ${{ github.event.inputs.exec_mode }} + EXEC_MODE: ${{ github.event.inputs.commitment_mode }} run: | set +e # Disable exit on error echo "Starting Erigon..." - # Only set ERIGON_EXEC3_PARALLEL on an explicit choice; otherwise ensure it's + # Only set ERIGON_COMMITMENT_PARALLEL on an explicit choice; otherwise ensure it's # unset so erigon uses its built-in default (and doesn't inherit runner env). if [ "$EXEC_MODE" = "parallel" ]; then - export ERIGON_EXEC3_PARALLEL=true - echo "Set ERIGON_EXEC3_PARALLEL=true (parallel mode)" + export ERIGON_COMMITMENT_PARALLEL=true + echo "Set ERIGON_COMMITMENT_PARALLEL=true (parallel mode)" elif [ "$EXEC_MODE" = "serial" ]; then - export ERIGON_EXEC3_PARALLEL=false - echo "Set ERIGON_EXEC3_PARALLEL=false (serial mode)" + export ERIGON_COMMITMENT_PARALLEL=false + echo "Set ERIGON_COMMITMENT_PARALLEL=false (serial mode)" else - unset ERIGON_EXEC3_PARALLEL - echo "Leaving ERIGON_EXEC3_PARALLEL unset (default behavior)" + unset ERIGON_COMMITMENT_PARALLEL + echo "Leaving ERIGON_COMMITMENT_PARALLEL unset (default behavior)" fi ./erigon --prune.mode=minimal --datadir $ERIGON_TESTBED_DATA_DIR --http.api admin,debug,eth,parity,erigon,trace,web3,txpool,ots,net --ws > erigon.log 2>&1 & @@ -378,7 +378,7 @@ jobs: --repo $client \ --branch $branch_name \ --commit $commit_hash \ - --test_name rpc-performance-test-latest${{ (matrix.client == 'erigon' && github.event.inputs.exec_mode == 'parallel' && '-parallel') || (matrix.client == 'erigon' && github.event.inputs.exec_mode == 'serial' && '-serial') || '' }}-$method \ + --test_name rpc-performance-test-latest${{ (matrix.client == 'erigon' && github.event.inputs.commitment_mode == 'parallel' && '-parallel') || (matrix.client == 'erigon' && github.event.inputs.commitment_mode == 'serial' && '-serial') || '' }}-$method \ --chain $CHAIN \ --runner ${{ runner.name }} \ --db_version $db_version \ @@ -451,7 +451,7 @@ jobs: if: (matrix.client == 'erigon' || needs.setup.outputs.run_geth == 'true') && steps.test_step.outputs.test_executed == 'true' uses: actions/upload-artifact@v7 with: - name: test-results-${{ env.CHAIN }}-${{ matrix.client }}${{ (matrix.client == 'erigon' && github.event.inputs.exec_mode == 'parallel' && '-parallel') || (matrix.client == 'erigon' && github.event.inputs.exec_mode == 'serial' && '-serial') || '' }} + name: test-results-${{ env.CHAIN }}-${{ matrix.client }}${{ (matrix.client == 'erigon' && github.event.inputs.commitment_mode == 'parallel' && '-parallel') || (matrix.client == 'erigon' && github.event.inputs.commitment_mode == 'serial' && '-serial') || '' }} path: ${{ env.past_test_dir }} - name: Stop Erigon @@ -471,7 +471,7 @@ jobs: if: matrix.client == 'erigon' && steps.test_step.outputs.test_executed == 'true' uses: actions/upload-artifact@v7 with: - name: erigon-logs-${{ env.CHAIN }}${{ (github.event.inputs.exec_mode == 'parallel' && '-parallel') || (github.event.inputs.exec_mode == 'serial' && '-serial') || '' }} + name: erigon-logs-${{ env.CHAIN }}${{ (github.event.inputs.commitment_mode == 'parallel' && '-parallel') || (github.event.inputs.commitment_mode == 'serial' && '-serial') || '' }} path: ${{ github.workspace }}/build/bin/erigon.log - name: Delete Erigon Testbed Data Directory diff --git a/.github/workflows/qa-rpc-performance-tests.yml b/.github/workflows/qa-rpc-performance-tests.yml index e08e7ab9d09..1303d34bfa2 100644 --- a/.github/workflows/qa-rpc-performance-tests.yml +++ b/.github/workflows/qa-rpc-performance-tests.yml @@ -11,8 +11,8 @@ on: type: boolean required: false default: false - exec_mode: - description: 'Erigon execution mode ("default" leaves ERIGON_EXEC3_PARALLEL unset)' + commitment_mode: + description: 'Erigon commitment mode ("default" leaves ERIGON_COMMITMENT_PARALLEL unset)' type: choice required: false default: default @@ -208,21 +208,21 @@ jobs: if: matrix.client == 'erigon' working-directory: ${{ github.workspace }}/build/bin env: - EXEC_MODE: ${{ github.event.inputs.exec_mode }} + EXEC_MODE: ${{ github.event.inputs.commitment_mode }} run: | echo "Starting RpcDaemon..." - # Only set ERIGON_EXEC3_PARALLEL on an explicit choice; otherwise ensure it's + # Only set ERIGON_COMMITMENT_PARALLEL on an explicit choice; otherwise ensure it's # unset so erigon uses its built-in default (and doesn't inherit runner env). if [ "$EXEC_MODE" = "parallel" ]; then - export ERIGON_EXEC3_PARALLEL=true - echo "Set ERIGON_EXEC3_PARALLEL=true (parallel mode)" + export ERIGON_COMMITMENT_PARALLEL=true + echo "Set ERIGON_COMMITMENT_PARALLEL=true (parallel mode)" elif [ "$EXEC_MODE" = "serial" ]; then - export ERIGON_EXEC3_PARALLEL=false - echo "Set ERIGON_EXEC3_PARALLEL=false (serial mode)" + export ERIGON_COMMITMENT_PARALLEL=false + echo "Set ERIGON_COMMITMENT_PARALLEL=false (serial mode)" else - unset ERIGON_EXEC3_PARALLEL - echo "Leaving ERIGON_EXEC3_PARALLEL unset (default behavior)" + unset ERIGON_COMMITMENT_PARALLEL + echo "Leaving ERIGON_COMMITMENT_PARALLEL unset (default behavior)" fi ./rpcdaemon --datadir $ERIGON_REFERENCE_DATA_DIR --http.api admin,debug,eth,parity,erigon,trace,web3,txpool,ots,net > erigon.log 2>&1 & @@ -371,7 +371,7 @@ jobs: --repo $client \ --branch $branch_name \ --commit $commit_hash \ - --test_name rpc-performance-test-$client${{ (matrix.client == 'erigon' && github.event.inputs.exec_mode == 'parallel' && '-parallel') || (matrix.client == 'erigon' && github.event.inputs.exec_mode == 'serial' && '-serial') || '' }}-$method \ + --test_name rpc-performance-test-$client${{ (matrix.client == 'erigon' && github.event.inputs.commitment_mode == 'parallel' && '-parallel') || (matrix.client == 'erigon' && github.event.inputs.commitment_mode == 'serial' && '-serial') || '' }}-$method \ --chain $CHAIN \ --runner ${{ runner.name }} \ --db_version $db_version \ @@ -434,7 +434,7 @@ jobs: if: always() && matrix.client == 'erigon' && steps.rpcdaemon_running_step.outputs.rpc_daemon_started == 'true' uses: actions/upload-artifact@v7 with: - name: rpcdaemon-logs${{ (github.event.inputs.exec_mode == 'parallel' && '-parallel') || (github.event.inputs.exec_mode == 'serial' && '-serial') || '' }} + name: rpcdaemon-logs${{ (github.event.inputs.commitment_mode == 'parallel' && '-parallel') || (github.event.inputs.commitment_mode == 'serial' && '-serial') || '' }} path: ${{ github.workspace }}/build/bin/erigon.log - name: Restore Erigon Chaindata Directory @@ -470,7 +470,7 @@ jobs: if: (matrix.client == 'erigon' || needs.setup.outputs.run_geth == 'true') && steps.test_step.outputs.test_executed == 'true' uses: actions/upload-artifact@v7 with: - name: test-results-${{ env.CHAIN }}-${{ matrix.client }}${{ (matrix.client == 'erigon' && github.event.inputs.exec_mode == 'parallel' && '-parallel') || (matrix.client == 'erigon' && github.event.inputs.exec_mode == 'serial' && '-serial') || '' }} + name: test-results-${{ env.CHAIN }}-${{ matrix.client }}${{ (matrix.client == 'erigon' && github.event.inputs.commitment_mode == 'parallel' && '-parallel') || (matrix.client == 'erigon' && github.event.inputs.commitment_mode == 'serial' && '-serial') || '' }} path: ${{ env.past_test_dir }} - name: Action to check failure condition diff --git a/.github/workflows/qa-stage-exec.yml b/.github/workflows/qa-stage-exec.yml index 97ceb6b514b..bd6cee0d6a7 100644 --- a/.github/workflows/qa-stage-exec.yml +++ b/.github/workflows/qa-stage-exec.yml @@ -17,48 +17,48 @@ concurrency: jobs: stage-exec-test: - name: stage-exec-test (${{ matrix.mode_name }}, ${{ matrix.exec_mode }}) + name: stage-exec-test (${{ matrix.mode_name }}, ${{ matrix.commitment_mode }}) runs-on: [self-hosted, qa, Ethereum, tip-tracking] strategy: fail-fast: false matrix: - # Each (mode_name, exec_mode) pair runs separately. The testbed + # Each (mode_name, commitment_mode) pair runs separately. The testbed # data dir already disambiguates by mode_name; we extend it with - # exec_mode so serial+parallel entries don't clobber each other. + # commitment_mode so serial+parallel entries don't clobber each other. include: - mode_name: resume-nonchaintip extra_flags: "" test_name: stage_exec_resume_nonchaintip - exec_mode: serial + commitment_mode: serial - mode_name: resume-nonchaintip extra_flags: "" test_name: stage_exec_resume_nonchaintip - exec_mode: parallel + commitment_mode: parallel - mode_name: from-0 extra_flags: "--rm-state-all" test_name: stage_exec_from_0 - exec_mode: serial + commitment_mode: serial - mode_name: from-0 extra_flags: "--rm-state-all" test_name: stage_exec_from_0 - exec_mode: parallel + commitment_mode: parallel - mode_name: chaintip extra_flags: "--chaintip" test_name: stage_exec_resume_chaintip - exec_mode: serial + commitment_mode: serial - mode_name: chaintip extra_flags: "--chaintip" test_name: stage_exec_resume_chaintip - exec_mode: parallel + commitment_mode: parallel env: - ERIGON_TESTBED_DATA_DIR: /opt/erigon-testbed/datadir-${{ matrix.mode_name }}-${{ matrix.exec_mode }} + ERIGON_TESTBED_DATA_DIR: /opt/erigon-testbed/datadir-${{ matrix.mode_name }}-${{ matrix.commitment_mode }} ERIGON_QA_PATH: /home/qarunner/erigon-qa TIMEOUT_SECONDS: 360 CHAIN: mainnet - # Toggle dbg.Exec3Parallel from CI without code changes. envLookup - # in common/dbg/dbg_env.go auto-prepends ERIGON_, so this maps to - # the EXEC3_PARALLEL flag declared in common/dbg/experiments.go. - ERIGON_EXEC3_PARALLEL: ${{ matrix.exec_mode == 'parallel' && 'true' || 'false' }} + # Toggle statecfg.ExperimentalParallelCommitment from CI without code + # changes. envLookup in common/dbg/dbg_env.go auto-prepends ERIGON_, so + # this maps to the COMMITMENT_PARALLEL flag read in db/state/statecfg. + ERIGON_COMMITMENT_PARALLEL: ${{ matrix.commitment_mode == 'parallel' && 'true' || 'false' }} steps: - name: Check out repository @@ -114,7 +114,7 @@ jobs: python3 $ERIGON_QA_PATH/test_system/qa-tests/stage-exec/run_and_check_stage_exec.py \ ${{ github.workspace }}/build/bin $ERIGON_TESTBED_DATA_DIR $TIMEOUT_SECONDS $CHAIN \ - --result-file ${{ github.workspace }}/result-$CHAIN-${{ matrix.mode_name }}-${{ matrix.exec_mode }}.json \ + --result-file ${{ github.workspace }}/result-$CHAIN-${{ matrix.mode_name }}-${{ matrix.commitment_mode }}.json \ ${{ matrix.extra_flags }} test_exit_status=$? @@ -135,7 +135,7 @@ jobs: TEST_RESULT: ${{ steps.test_step.outputs.TEST_RESULT }} TEST_NAME: ${{ matrix.test_name }} MODE_NAME: ${{ matrix.mode_name }} - EXEC_MODE: ${{ matrix.exec_mode }} + COMMITMENT_MODE: ${{ matrix.commitment_mode }} run: | db_version=$(python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/prod_info.py $ERIGON_REFERENCE_DATA_DIR/../production.ini production erigon_repo_commit) if [ -z "$db_version" ]; then @@ -151,20 +151,20 @@ jobs: --runner "$RUNNER_NAME" \ --db_version $db_version \ --outcome $TEST_RESULT \ - --result_file "$GITHUB_WORKSPACE/result-$CHAIN-$MODE_NAME-$EXEC_MODE.json" + --result_file "$GITHUB_WORKSPACE/result-$CHAIN-$MODE_NAME-$COMMITMENT_MODE.json" - name: Upload test results if: ${{ always() && steps.test_step.outputs.test_executed == 'true' }} uses: actions/upload-artifact@v7 with: - name: test-results-${{ matrix.mode_name }}-${{ matrix.exec_mode }} - path: ${{ github.workspace }}/result-${{ env.CHAIN }}-${{ matrix.mode_name }}-${{ matrix.exec_mode }}.json + name: test-results-${{ matrix.mode_name }}-${{ matrix.commitment_mode }} + path: ${{ github.workspace }}/result-${{ env.CHAIN }}-${{ matrix.mode_name }}-${{ matrix.commitment_mode }}.json - name: Upload erigon logs if: ${{ always() && steps.test_step.outputs.test_executed == 'true' }} uses: actions/upload-artifact@v7 with: - name: erigon-logs-${{ matrix.mode_name }}-${{ matrix.exec_mode }} + name: erigon-logs-${{ matrix.mode_name }}-${{ matrix.commitment_mode }} path: ${{ env.ERIGON_TESTBED_DATA_DIR }}/logs/ - name: Delete Erigon Testbed Data Directory diff --git a/.github/workflows/qa-txpool-performance-test.yml b/.github/workflows/qa-txpool-performance-test.yml index 57dc5ecce98..fab477c46ab 100644 --- a/.github/workflows/qa-txpool-performance-test.yml +++ b/.github/workflows/qa-txpool-performance-test.yml @@ -18,23 +18,23 @@ on: jobs: tx_pool_assertoor_test: - name: tx_pool_assertoor_test (${{ matrix.exec_mode }}) + name: tx_pool_assertoor_test (${{ matrix.commitment_mode }}) runs-on: [self-hosted, qa, X64, long-running] strategy: fail-fast: false # Self-hosted long-running pool is small; matrix entries serialize. max-parallel: 1 matrix: - exec_mode: + commitment_mode: - serial - parallel env: ERIGON_QA_PATH: /home/qarunner/erigon-qa - ENCLAVE_NAME: "kurtosis-run-${{ github.run_id }}-${{ matrix.exec_mode }}" - # Toggle dbg.Exec3Parallel from CI without code changes. envLookup - # in common/dbg/dbg_env.go auto-prepends ERIGON_, so this maps to - # the EXEC3_PARALLEL flag declared in common/dbg/experiments.go. - ERIGON_EXEC3_PARALLEL: ${{ matrix.exec_mode == 'parallel' && 'true' || 'false' }} + ENCLAVE_NAME: "kurtosis-run-${{ github.run_id }}-${{ matrix.commitment_mode }}" + # Toggle statecfg.ExperimentalParallelCommitment from CI without code + # changes. envLookup in common/dbg/dbg_env.go auto-prepends ERIGON_, so + # this maps to the COMMITMENT_PARALLEL flag read in db/state/statecfg. + ERIGON_COMMITMENT_PARALLEL: ${{ matrix.commitment_mode == 'parallel' && 'true' || 'false' }} steps: - name: Fast checkout git repository @@ -50,14 +50,14 @@ jobs: run: | docker build -t test/erigon:current-base . - - name: Bake exec_mode env into the runtime image - # Add ERIGON_EXEC3_PARALLEL as an ENV layer on top of the base image - # so the matrix entries differ in exec mode. ENV layers are cheap and - # don't invalidate earlier layer caches. + - name: Bake commitment_mode env into the runtime image + # Add ERIGON_COMMITMENT_PARALLEL as an ENV layer on top of the base + # image so the matrix entries differ in commitment mode. ENV layers are + # cheap and don't invalidate earlier layer caches. run: | docker build -t test/erigon:current - <> clients/erigon/Dockerfile + echo "ENV ERIGON_COMMITMENT_PARALLEL=${ERIGON_COMMITMENT_PARALLEL}" >> clients/erigon/Dockerfile erigon_extra_flags="${{ matrix.erigon-extra-flags }}" if [ -n "$erigon_extra_flags" ]; then echo "Patching erigon.sh with extra flags: $erigon_extra_flags" @@ -302,9 +303,9 @@ jobs: if: always() uses: actions/upload-artifact@v7 with: - # exec_mode in the artifact name keeps the two matrix entries from + # commitment_mode in the artifact name keeps the two matrix entries from # clobbering each other's logs on the same artifact key. - name: hive-workspace-log-${{ matrix.shard }}-${{ matrix.exec_mode }} + name: hive-workspace-log-${{ matrix.shard }}-${{ matrix.commitment_mode }} path: hive/workspace/logs if-no-files-found: ignore - name: Test Results diff --git a/.github/workflows/test-hive.yml b/.github/workflows/test-hive.yml index e79e0e7a458..d591a295b93 100644 --- a/.github/workflows/test-hive.yml +++ b/.github/workflows/test-hive.yml @@ -12,7 +12,7 @@ concurrency: jobs: test-hive: - name: test-hive (${{ matrix.sim }}, ${{ matrix.sim-limit }}, ${{ matrix.exec_mode }}) + name: test-hive (${{ matrix.sim }}, ${{ matrix.sim-limit }}, ${{ matrix.commitment_mode }}) if: >- ${{ !github.event.pull_request.number || (!github.event.pull_request.draft @@ -26,10 +26,11 @@ jobs: # picture across every shard. fail-fast: ${{ github.event_name == 'merge_group' }} matrix: - # Each (sim, sim-limit) pair is run twice — once with serial exec - # (ERIGON_EXEC3_PARALLEL=false) and once with parallel — so engine-API - # / wire-protocol divergence between the two paths is caught on the - # PR. Matrix entries spawn separate `hive` group runners and run + # Each (sim, sim-limit) pair is run twice — once with serial commitment + # (ERIGON_COMMITMENT_PARALLEL=false) and once with parallel — so + # engine-API / wire-protocol divergence between the two paths is caught + # on the PR. Execution is parallel in both legs (default). Matrix + # entries spawn separate `hive` group runners and run # concurrently — wall-clock unchanged, runner-minutes doubled. # `sim` is the simulator path passed to `hive --sim`. Most simulators # live under simulators/ethereum/, but a few (e.g. devp2p) are top-level. @@ -37,53 +38,53 @@ jobs: - sim: ethereum/engine sim-limit: exchange-capabilities|auth max-allowed-failures: 0 - exec_mode: serial + commitment_mode: serial - sim: ethereum/engine sim-limit: exchange-capabilities|auth max-allowed-failures: 0 - exec_mode: parallel + commitment_mode: parallel - sim: ethereum/engine sim-limit: withdrawals max-allowed-failures: 0 - exec_mode: serial + commitment_mode: serial - sim: ethereum/engine sim-limit: withdrawals max-allowed-failures: 0 - exec_mode: parallel + commitment_mode: parallel - sim: ethereum/engine sim-limit: cancun max-allowed-failures: 0 - exec_mode: serial + commitment_mode: serial - sim: ethereum/engine sim-limit: cancun max-allowed-failures: 0 - exec_mode: parallel + commitment_mode: parallel - sim: ethereum/engine sim-limit: api max-allowed-failures: 0 - exec_mode: serial + commitment_mode: serial - sim: ethereum/engine sim-limit: api max-allowed-failures: 0 - exec_mode: parallel + commitment_mode: parallel - sim: ethereum/rpc-compat sim-limit: ".*" max-allowed-failures: 0 - exec_mode: serial + commitment_mode: serial - sim: ethereum/rpc-compat sim-limit: ".*" max-allowed-failures: 0 - exec_mode: parallel + commitment_mode: parallel - sim: devp2p sim-limit: eth max-allowed-failures: 0 - exec_mode: serial + commitment_mode: serial # discv5 exercises peer discovery, not the EL exec path, so it runs in - # just one exec mode — duplicating it in the serial leg adds no signal. + # just one commitment mode — duplicating it in the serial leg adds no signal. - sim: devp2p sim-limit: eth|discv5 max-allowed-failures: 0 - exec_mode: parallel + commitment_mode: parallel steps: - name: Checkout Erigon uses: actions/checkout@v7 @@ -149,10 +150,10 @@ jobs: - name: Get dependencies and build hive env: EXECUTION_APIS_REF: ${{ steps.hive-version.outputs.execution_apis_ref }} - # Toggle dbg.Exec3Parallel inside the hive erigon container. - # We bake this as an ENV directive into the client Dockerfile so - # every erigon instance hive launches inherits it. - ERIGON_EXEC3_PARALLEL: ${{ matrix.exec_mode == 'parallel' && 'true' || 'false' }} + # Toggle statecfg.ExperimentalParallelCommitment inside the hive + # erigon container. We bake this as an ENV directive into the client + # Dockerfile so every erigon instance hive launches inherits it. + ERIGON_COMMITMENT_PARALLEL: ${{ matrix.commitment_mode == 'parallel' && 'true' || 'false' }} run: | cd hive retry() { @@ -175,10 +176,10 @@ jobs: echo "ERROR: failed to repoint hive's erigon client Dockerfile at hive/erigon:cilocal" exit 1 fi - # Inject ERIGON_EXEC3_PARALLEL into the runtime image so the + # Inject ERIGON_COMMITMENT_PARALLEL into the runtime image so the # erigon process inside hive picks it up. Append as the last layer # so it doesn't invalidate earlier build caches. - echo "ENV ERIGON_EXEC3_PARALLEL=${ERIGON_EXEC3_PARALLEL}" >> clients/erigon/Dockerfile + echo "ENV ERIGON_COMMITMENT_PARALLEL=${ERIGON_COMMITMENT_PARALLEL}" >> clients/erigon/Dockerfile # Pin the execution-apis ref used by the rpc-compat simulator so that # upstream test additions don't break CI unexpectedly. # SECURITY: value comes from hive-versions.json which fork PRs can modify; @@ -254,13 +255,13 @@ jobs: - name: Compute artifact name id: artifact-name env: - RAW_NAME: hive-workspace-log-${{ matrix.sim }}-${{ matrix.sim-limit }}-${{ matrix.exec_mode }} + RAW_NAME: hive-workspace-log-${{ matrix.sim }}-${{ matrix.sim-limit }}-${{ matrix.commitment_mode }} run: echo "name=${RAW_NAME//[^A-Za-z0-9._-]/_}" >> "$GITHUB_OUTPUT" - name: Upload output log uses: actions/upload-artifact@v7 with: - # exec_mode in the artifact name keeps the two matrix entries from + # commitment_mode in the artifact name keeps the two matrix entries from # clobbering each other's logs on the same artifact key. name: ${{ steps.artifact-name.outputs.name }} path: hive/workspace/logs diff --git a/.github/workflows/test-kurtosis-assertoor.yml b/.github/workflows/test-kurtosis-assertoor.yml index 37bf448da6a..af645d30295 100644 --- a/.github/workflows/test-kurtosis-assertoor.yml +++ b/.github/workflows/test-kurtosis-assertoor.yml @@ -304,7 +304,7 @@ jobs: key: docker-buildkit-${{ env.BUILDKIT_IMAGE }} assertoor_test: - name: assertoor_${{ matrix.suite }}_${{ matrix.exec_mode }}_test + name: assertoor_${{ matrix.suite }}_${{ matrix.commitment_mode }}_test needs: build-erigon-image # On cache-warming runs build-erigon-image alone warms the kurtosis image # cache; there's nothing for the matrix to do (the test step is skipped). @@ -324,32 +324,33 @@ jobs: # pass but pathological hangs (e.g. a node that silently stops proposing) # fail in reasonable time instead of burning hours. # - # Each suite runs twice — once with serial exec - # (ERIGON_EXEC3_PARALLEL=false) and once with parallel — so divergence - # is caught on the PR. Matrix entries spawn separate hosted runners - # and run concurrently — wall-clock unchanged, runner-minutes doubled. + # Each suite runs twice — once with serial commitment + # (ERIGON_COMMITMENT_PARALLEL=false) and once with parallel — so + # divergence is caught on the PR. Execution is parallel in both legs + # (default). Matrix entries spawn separate hosted runners and run + # concurrently — wall-clock unchanged, runner-minutes doubled. # All entries share the single erigon image built by build-erigon-image. include: - suite: regular package_args: .github/workflows/kurtosis/regular-assertoor.io ethereum_package_branch: "5.0.1" test_timeout_minutes: 50 - exec_mode: serial + commitment_mode: serial - suite: regular package_args: .github/workflows/kurtosis/regular-assertoor.io ethereum_package_branch: "5.0.1" test_timeout_minutes: 50 - exec_mode: parallel + commitment_mode: parallel - suite: pectra package_args: .github/workflows/kurtosis/pectra.io ethereum_package_branch: "5.0.1" test_timeout_minutes: 45 - exec_mode: serial + commitment_mode: serial - suite: pectra package_args: .github/workflows/kurtosis/pectra.io ethereum_package_branch: "5.0.1" test_timeout_minutes: 45 - exec_mode: parallel + commitment_mode: parallel - suite: glamsterdam package_args: .github/workflows/kurtosis/glamsterdam.io # Pinned to 6.1.0 rather than main: commit 835dd9b on main introduced GpuConfig, @@ -357,19 +358,19 @@ jobs: # Unpin to main once CI is upgraded to Kurtosis 1.18.1. ethereum_package_branch: "6.1.0" test_timeout_minutes: 20 - exec_mode: parallel + commitment_mode: parallel - suite: caplin-minimal package_args: .github/workflows/kurtosis/caplin-minimal-assertoor.io ethereum_package_url: "github.com/erigontech/ethereum-package" ethereum_package_branch: "erigontech/fix-caplin-launcher" test_timeout_minutes: 20 - exec_mode: serial + commitment_mode: serial - suite: caplin-minimal package_args: .github/workflows/kurtosis/caplin-minimal-assertoor.io ethereum_package_url: "github.com/erigontech/ethereum-package" ethereum_package_branch: "erigontech/fix-caplin-launcher" test_timeout_minutes: 20 - exec_mode: parallel + commitment_mode: parallel steps: - name: Fast checkout git repository @@ -502,16 +503,16 @@ jobs: - name: Load erigon base image into daemon run: docker load -i "${RUNNER_TEMP}/erigon-base-image.tar" - - name: Bake exec_mode env into the runtime image + - name: Bake commitment_mode env into the runtime image # Kurtosis launches `test/erigon:current` as the EL participant. - # Add ERIGON_EXEC3_PARALLEL as an ENV layer on top of the shared - # base image so the matrix entries differ only in exec mode. + # Add ERIGON_COMMITMENT_PARALLEL as an ENV layer on top of the shared + # base image so the matrix entries differ only in commitment mode. env: - ERIGON_EXEC3_PARALLEL: ${{ matrix.exec_mode == 'parallel' && 'true' || 'false' }} + ERIGON_COMMITMENT_PARALLEL: ${{ matrix.commitment_mode == 'parallel' && 'true' || 'false' }} run: | docker build -t test/erigon:current - <&2 exit 2 fi -IFS=$'\t' read -r default_workers default_max exec3_parallel run_regex <<<"$shard_row" -# Always set ERIGON_EXEC3_PARALLEL explicitly (true or false) so the shard's -# behaviour is pinned to the manifest, independent of whatever dbg.Exec3Parallel -# defaults to at runtime. If the default flips, the shards still run the mode -# they were defined for. -export ERIGON_EXEC3_PARALLEL="$exec3_parallel" +IFS=$'\t' read -r default_workers default_max commitment_parallel run_regex <<<"$shard_row" +# Always set ERIGON_COMMITMENT_PARALLEL explicitly (true or false) so the shard's +# commitment mode is pinned to the manifest, independent of whatever +# statecfg.ExperimentalParallelCommitment defaults to at runtime. Execution is +# parallel in every shard (dbg.Exec3Parallel defaults true). +export ERIGON_COMMITMENT_PARALLEL="$commitment_parallel" # Strip "-parallel" / "-sequential" suffix for case-arm routing — both variants # share the same fixture path / regex as the parent shard; only the -# ERIGON_EXEC3_PARALLEL env var differs. +# ERIGON_COMMITMENT_PARALLEL env var differs. shard_route="${shard%-parallel}" shard_route="${shard_route%-sequential}" @@ -186,7 +187,7 @@ echo "max-allowed-failures: $max" # code 66: the Go race runtime's "data race detected" signal, emitted even when # the run completes and the JSON parses clean, so it must be checked explicitly. # The grep filter strips any init-time log lines (e.g. dbg.envLookup's -# "[WARN] [env]" message when ERIGON_EXEC3_PARALLEL is set fires before cmd/evm +# "[WARN] [env]" message when ERIGON_COMMITMENT_PARALLEL is set fires before cmd/evm # sets the log handler, and the default log handler writes to stdout) so jq # sees only JSON. raw_file=$(mktemp) From 9c4d5b286b32bb3a50dfdd74a9561c197aa62298 Mon Sep 17 00:00:00 2001 From: taratorio <94537774+taratorio@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:00:41 +0200 Subject: [PATCH 6/9] ci: pair legacy race shards by commitment mode --- .claude/skills/erigon-test-all/SKILL.md | 21 +++--- .claude/skills/erigon-test-race/SKILL.md | 8 +- tools/eest-spec-shards.yml | 95 +++++++++++++++++++++--- tools/run-eest-spec-test.sh | 17 +++-- 4 files changed, 111 insertions(+), 30 deletions(-) diff --git a/.claude/skills/erigon-test-all/SKILL.md b/.claude/skills/erigon-test-all/SKILL.md index 7ef1c48dcd5..7451c1f442c 100644 --- a/.claude/skills/erigon-test-all/SKILL.md +++ b/.claude/skills/erigon-test-all/SKILL.md @@ -15,27 +15,30 @@ To exercise the EEST suites locally, see `erigon-eest-spec` (or run a specific s ```bash make eest-spec-statetests-stable # state tests vs eest_stable fixtures -make eest-spec-blocktests-stable-sequential # blockchain tests vs eest_stable fixtures (ERIGON_EXEC3_PARALLEL=false) -make eest-spec-blocktests-stable-parallel # same, but with ERIGON_EXEC3_PARALLEL=true -make eest-spec-enginextests-stable-sequential # engine-x tests vs eest_stable (ERIGON_EXEC3_PARALLEL=false) -make eest-spec-enginextests-stable-parallel # same, but with ERIGON_EXEC3_PARALLEL=true +make eest-spec-blocktests-stable-sequential # blockchain tests vs eest_stable fixtures (serial commitment) +make eest-spec-blocktests-stable-parallel # same, but with parallel commitment +make eest-spec-enginextests-stable-sequential # engine-x tests vs eest_stable (serial commitment) +make eest-spec-enginextests-stable-parallel # same, but with parallel commitment make eest-spec-statetests-devnet # …vs eest_devnet fixtures -make eest-spec-blocktests-devnet # devnet blocktests (always parallel exec3) +make eest-spec-blocktests-devnet # devnet blocktests (serial commitment) make eest-spec-statetests-legacy # pinned legacy Cancun state-test archive make eest-spec-blocktests-legacy-consensus-sequential # Hive consensus fixture selection; - # -parallel and -race variants too + # -parallel and + # -race-{sequential,parallel} variants too make eest-spec-blocktests-legacy-constantinople-sequential # Hive legacy fixture selection; # -parallel plus three race partitions: # ...-race-constantinople, # ...-race-constantinople-fix, and - # ...-race-other-forks + # ...-race-other-forks; each race + # partition has sequential/parallel variants make eest-spec-blocktests-legacy-cancun-sequential # Hive legacy-cancun selection; # -parallel plus six race partitions: # ...-race-{berlin,shanghai,cancun, - # london,paris,other-forks} + # london,paris,other-forks}; each race + # partition has sequential/parallel variants make eest-spec-enginextests-benchmark-1m-sequential # engine-x benchmark fixtures @ 1M gas target # (with per-test --time stats); @@ -50,7 +53,7 @@ make eest-spec-blocktests-stable-race-cancun-sequential # (e.g. ...-race-cancun-{sequential,parallel}) ``` -The shard list / failure budgets / `exec3-parallel` flags live in `tools/eest-spec-shards.yml` (single source of truth for both this workflow and `tools/run-eest-spec-test.sh`). See `EEST_SPEC_SHARDS` / `EEST_SPEC_RACE_SHARDS` in the root `Makefile` for the partition into race vs non-race targets. +The shard list / failure budgets / `commitment-parallel` flags live in `tools/eest-spec-shards.yml` (single source of truth for both this workflow and `tools/run-eest-spec-test.sh`). See `EEST_SPEC_SHARDS` / `EEST_SPEC_RACE_SHARDS` in the root `Makefile` for the partition into race vs non-race targets. **Pitfall: stale `evm` / `evm.race` binary.** Always invoke shards via `make eest-spec-` — the Makefile lists `evm` (or `evm.race`) as a prereq and `go build` is cache-aware, so a stale binary gets rebuilt automatically. Calling `bash tools/run-eest-spec-test.sh ` directly **bypasses** the rebuild and silently exercises whatever `build/bin/evm{,.race}` happens to be on disk against current fixtures, inflating failures or hiding regressions. After pulling code, switching branches, or any time you suspect the binary is older than HEAD: `rm -f build/bin/evm build/bin/evm.race && make evm evm.race` before re-running. diff --git a/.claude/skills/erigon-test-race/SKILL.md b/.claude/skills/erigon-test-race/SKILL.md index 8d720d6e5de..61e32d2e142 100644 --- a/.claude/skills/erigon-test-race/SKILL.md +++ b/.claude/skills/erigon-test-race/SKILL.md @@ -16,12 +16,12 @@ Use the dedicated EEST blocktest race shards: ```bash make eest-spec-blocktests-stable-race-{pre-cancun,cancun,prague,osaka}-{sequential,parallel} make eest-spec-blocktests-devnet-race-amsterdam -make eest-spec-blocktests-legacy-consensus-race -make eest-spec-blocktests-legacy-constantinople-race-{constantinople,constantinople-fix,other-forks} -make eest-spec-blocktests-legacy-cancun-race-{berlin,shanghai,cancun,london,paris,other-forks} +make eest-spec-blocktests-legacy-consensus-race-{sequential,parallel} +make eest-spec-blocktests-legacy-constantinople-race-{constantinople,constantinople-fix,other-forks}-{sequential,parallel} +make eest-spec-blocktests-legacy-cancun-race-{berlin,shanghai,cancun,london,paris,other-forks}-{sequential,parallel} ``` -These targets build a race-instrumented `evm.race` binary automatically (see `EEST_SPEC_RACE_SHARDS` in the root `Makefile`). The stable `-sequential` / `-parallel` pairs pin both execution modes; the devnet and legacy race shards pin parallel execution to match their fixture topology. For the consensus spec suite or other Go packages, pass `GOFLAGS='-race'` or invoke `go test -race` against the relevant package directly. +These targets build a race-instrumented `evm.race` binary automatically (see `EEST_SPEC_RACE_SHARDS` in the root `Makefile`). The stable and legacy `-sequential` / `-parallel` pairs pin both commitment modes; execution remains parallel in every pair. The unsuffixed devnet race shard uses serial commitment. For the consensus spec suite or other Go packages, pass `GOFLAGS='-race'` or invoke `go test -race` against the relevant package directly. **Pitfall: stale `evm.race` binary.** `make eest-spec-` lists `evm.race` as a prereq and `go build` is cache-aware, so a stale binary gets rebuilt. Calling `bash tools/run-eest-spec-test.sh ` directly with `EVM_BIN=build/bin/evm.race` **bypasses** the rebuild and silently runs an old race-instrumented binary against current fixtures — race reports against code that no longer exists, missed races against code that does. After pulling or switching branches: `rm -f build/bin/evm.race && make evm.race` before re-running. diff --git a/tools/eest-spec-shards.yml b/tools/eest-spec-shards.yml index 947c60c8717..e1fc82bdfdf 100644 --- a/tools/eest-spec-shards.yml +++ b/tools/eest-spec-shards.yml @@ -74,7 +74,13 @@ commitment-parallel: true exclude: - '/\.meta/' -- shard: blocktests-legacy-consensus-race +- shard: blocktests-legacy-consensus-race-sequential + workers: 12 + max-allowed-failures: 0 + expected-tests: 1142 + exclude: + - '/\.meta/' +- shard: blocktests-legacy-consensus-race-parallel workers: 12 max-allowed-failures: 0 expected-tests: 1142 @@ -96,7 +102,15 @@ commitment-parallel: true exclude: - '/\.meta/' -- shard: blocktests-legacy-constantinople-race-constantinople +- shard: blocktests-legacy-constantinople-race-constantinople-sequential + workers: 12 + # PoW fork-choice and total-difficulty support: https://github.com/erigontech/erigon/issues/22061 + max-allowed-failures: 5 + expected-tests: 10807 + run: '_Constantinople$' + exclude: + - '/\.meta/' +- shard: blocktests-legacy-constantinople-race-constantinople-parallel workers: 12 # PoW fork-choice and total-difficulty support: https://github.com/erigontech/erigon/issues/22061 max-allowed-failures: 5 @@ -105,7 +119,15 @@ run: '_Constantinople$' exclude: - '/\.meta/' -- shard: blocktests-legacy-constantinople-race-constantinople-fix +- shard: blocktests-legacy-constantinople-race-constantinople-fix-sequential + workers: 12 + # PoW fork-choice and total-difficulty support: https://github.com/erigontech/erigon/issues/22061 + max-allowed-failures: 5 + expected-tests: 10802 + run: '_ConstantinopleFix$' + exclude: + - '/\.meta/' +- shard: blocktests-legacy-constantinople-race-constantinople-fix-parallel workers: 12 # PoW fork-choice and total-difficulty support: https://github.com/erigontech/erigon/issues/22061 max-allowed-failures: 5 @@ -114,7 +136,15 @@ run: '_ConstantinopleFix$' exclude: - '/\.meta/' -- shard: blocktests-legacy-constantinople-race-other-forks +- shard: blocktests-legacy-constantinople-race-other-forks-sequential + workers: 12 + # PoW fork-choice and total-difficulty support: https://github.com/erigontech/erigon/issues/22061 + max-allowed-failures: 14 + expected-tests: 11006 + exclude: + - '/\.meta/' + - '::.*_Constantinople(Fix)?$' +- shard: blocktests-legacy-constantinople-race-other-forks-parallel workers: 12 # PoW fork-choice and total-difficulty support: https://github.com/erigontech/erigon/issues/22061 max-allowed-failures: 14 @@ -138,7 +168,15 @@ commitment-parallel: true exclude: - '/\.meta/' -- shard: blocktests-legacy-cancun-race-berlin +- shard: blocktests-legacy-cancun-race-berlin-sequential + workers: 12 + # PoW fork-choice and total-difficulty support: https://github.com/erigontech/erigon/issues/22061 + max-allowed-failures: 7 + expected-tests: 14026 + run: '(_Berlin$|fork_Berlin-)' + exclude: + - '/\.meta/' +- shard: blocktests-legacy-cancun-race-berlin-parallel workers: 12 # PoW fork-choice and total-difficulty support: https://github.com/erigontech/erigon/issues/22061 max-allowed-failures: 7 @@ -147,7 +185,14 @@ run: '(_Berlin$|fork_Berlin-)' exclude: - '/\.meta/' -- shard: blocktests-legacy-cancun-race-shanghai +- shard: blocktests-legacy-cancun-race-shanghai-sequential + workers: 12 + max-allowed-failures: 0 + expected-tests: 20689 + run: '(_Shanghai$|fork_Shanghai-)' + exclude: + - '/\.meta/' +- shard: blocktests-legacy-cancun-race-shanghai-parallel workers: 12 max-allowed-failures: 0 expected-tests: 20689 @@ -155,7 +200,14 @@ run: '(_Shanghai$|fork_Shanghai-)' exclude: - '/\.meta/' -- shard: blocktests-legacy-cancun-race-cancun +- shard: blocktests-legacy-cancun-race-cancun-sequential + workers: 12 + max-allowed-failures: 0 + expected-tests: 21849 + run: '(_Cancun$|fork_Cancun-)' + exclude: + - '/\.meta/' +- shard: blocktests-legacy-cancun-race-cancun-parallel workers: 12 max-allowed-failures: 0 expected-tests: 21849 @@ -163,7 +215,15 @@ run: '(_Cancun$|fork_Cancun-)' exclude: - '/\.meta/' -- shard: blocktests-legacy-cancun-race-london +- shard: blocktests-legacy-cancun-race-london-sequential + workers: 12 + # PoW fork-choice and total-difficulty support: https://github.com/erigontech/erigon/issues/22061 + max-allowed-failures: 7 + expected-tests: 20337 + run: '(_London$|fork_London-)' + exclude: + - '/\.meta/' +- shard: blocktests-legacy-cancun-race-london-parallel workers: 12 # PoW fork-choice and total-difficulty support: https://github.com/erigontech/erigon/issues/22061 max-allowed-failures: 7 @@ -172,7 +232,14 @@ run: '(_London$|fork_London-)' exclude: - '/\.meta/' -- shard: blocktests-legacy-cancun-race-paris +- shard: blocktests-legacy-cancun-race-paris-sequential + workers: 12 + max-allowed-failures: 0 + expected-tests: 20369 + run: '(_Paris$|fork_Paris-)' + exclude: + - '/\.meta/' +- shard: blocktests-legacy-cancun-race-paris-parallel workers: 12 max-allowed-failures: 0 expected-tests: 20369 @@ -180,7 +247,15 @@ run: '(_Paris$|fork_Paris-)' exclude: - '/\.meta/' -- shard: blocktests-legacy-cancun-race-other-forks +- shard: blocktests-legacy-cancun-race-other-forks-sequential + workers: 12 + # PoW fork-choice and total-difficulty support: https://github.com/erigontech/erigon/issues/22061 + max-allowed-failures: 12 + expected-tests: 14713 + exclude: + - '/\.meta/' + - '::.*(_(Berlin|Shanghai|Cancun|London|Paris)$|fork_(Berlin|Shanghai|Cancun|London|Paris)-)' +- shard: blocktests-legacy-cancun-race-other-forks-parallel workers: 12 # PoW fork-choice and total-difficulty support: https://github.com/erigontech/erigon/issues/22061 max-allowed-failures: 12 diff --git a/tools/run-eest-spec-test.sh b/tools/run-eest-spec-test.sh index 6bca2207992..5159f11ee9c 100755 --- a/tools/run-eest-spec-test.sh +++ b/tools/run-eest-spec-test.sh @@ -10,18 +10,21 @@ # statetests-legacy complete legacy Cancun state tests # blocktests-stable-sequential blockchain tests vs. eest_stable # blocktests-devnet blockchain tests vs. eest_devnet -# blocktests-legacy-consensus-{sequential,parallel,race} +# blocktests-legacy-consensus-{sequential,parallel} +# Hive consensus suite +# blocktests-legacy-consensus-race-{sequential,parallel} +# race-detector variants of the # Hive consensus suite # blocktests-legacy-constantinople-{sequential,parallel} # Hive legacy suite -# blocktests-legacy-constantinople-race-{constantinople,constantinople-fix,other-forks} -# race-detector partition of the -# Hive legacy suite +# blocktests-legacy-constantinople-race-* +# three fork partitions, each with +# sequential/parallel commitment # blocktests-legacy-cancun-{sequential,parallel} # Hive legacy-cancun suite -# blocktests-legacy-cancun-race-{berlin,shanghai,cancun,london,paris,other-forks} -# race-detector partition of the -# Hive legacy-cancun suite +# blocktests-legacy-cancun-race-* +# six fork partitions, each with +# sequential/parallel commitment # enginextests-stable-sequential engine-x tests vs. eest_stable # enginextests-benchmark-{1m,5m,10m,30m,60m,100m,150m}-sequential # engine-x benchmark fixtures per From 163d91e3936fba452e78413f33ebb1eed0c9d5a3 Mon Sep 17 00:00:00 2001 From: taratorio <94537774+taratorio@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:34:51 +0200 Subject: [PATCH 7/9] ci, db: exercise serial and parallel commitment --- .github/workflows/test-eest-spec.yml | 1 + .github/workflows/test-hive-eest.yml | 8 +++ .github/workflows/test-kurtosis-assertoor.yml | 8 +++ db/state/aggregator_bench_test.go | 2 + db/test/domain_shared_bench_test.go | 18 ++++--- db/test/lifecycle_bench_test.go | 16 ++---- tools/eest-spec-shards.yml | 52 +++++++++++++++++-- tools/run-eest-spec-test.sh | 12 +++-- 8 files changed, 94 insertions(+), 23 deletions(-) diff --git a/.github/workflows/test-eest-spec.yml b/.github/workflows/test-eest-spec.yml index 8c13cd1b557..5c416bdd510 100644 --- a/.github/workflows/test-eest-spec.yml +++ b/.github/workflows/test-eest-spec.yml @@ -122,6 +122,7 @@ jobs: env: EEST_SPEC_MAX_FAILURES: ${{ matrix.max-allowed-failures }} EEST_SPEC_WORKERS: ${{ matrix.workers }} + ERIGON_COMMITMENT_PARALLEL: ${{ matrix.commitment-parallel }} SHARD: ${{ matrix.shard }} run: make "eest-spec-$SHARD" diff --git a/.github/workflows/test-hive-eest.yml b/.github/workflows/test-hive-eest.yml index 93e99aae3b7..0093deaaa66 100644 --- a/.github/workflows/test-hive-eest.yml +++ b/.github/workflows/test-hive-eest.yml @@ -115,6 +115,14 @@ jobs: max-failures: 0 commitment_mode: parallel # Glamsterdam devnet: BAL EIPs against devnet fixtures + - sim: consume-engine + sim-limit: ".*(2780|7708|7778|7843|7928|7954|7976|7981|7997|8024|8037|8038|8246|8282).*" + shard: glamsterdam-devnet + fixtures-tarball: eest_devnet + extra-hive-flags: "--sim.loglevel=3 --client.checktimelimit=300s" + erigon-extra-flags: "--experimental.bal" + max-failures: 0 + commitment_mode: serial - sim: consume-engine sim-limit: ".*(2780|7708|7778|7843|7928|7954|7976|7981|7997|8024|8037|8038|8246|8282).*" shard: glamsterdam-devnet diff --git a/.github/workflows/test-kurtosis-assertoor.yml b/.github/workflows/test-kurtosis-assertoor.yml index 89ab75cc90d..a759807b009 100644 --- a/.github/workflows/test-kurtosis-assertoor.yml +++ b/.github/workflows/test-kurtosis-assertoor.yml @@ -363,6 +363,14 @@ jobs: ethereum_package_branch: "5.0.1" test_timeout_minutes: 45 commitment_mode: parallel + - suite: glamsterdam + package_args: .github/workflows/kurtosis/glamsterdam.io + # Pinned to 6.1.0 rather than main: commit 835dd9b on main introduced GpuConfig, + # a Starlark built-in that requires Kurtosis CLI >=1.18.1, breaking Starlark eval. + # Unpin to main once CI is upgraded to Kurtosis 1.18.1. + ethereum_package_branch: "6.1.0" + test_timeout_minutes: 20 + commitment_mode: serial - suite: glamsterdam package_args: .github/workflows/kurtosis/glamsterdam.io # Pinned to 6.1.0 rather than main: commit 835dd9b on main introduced GpuConfig, diff --git a/db/state/aggregator_bench_test.go b/db/state/aggregator_bench_test.go index 418f9eb98d5..789b9391091 100644 --- a/db/state/aggregator_bench_test.go +++ b/db/state/aggregator_bench_test.go @@ -70,6 +70,8 @@ func BenchmarkAggregator_Processing(b *testing.B) { domains, err := execctx.NewSharedDomains(ctx, tx, log.New()) require.NoError(b, err) defer domains.Close() + domains.EnableParaTrieDB(db) + require.Equal(b, execctx.PickTrieVariant(), domains.GetCommitmentCtx().Trie().Variant()) b.ReportAllocs() diff --git a/db/test/domain_shared_bench_test.go b/db/test/domain_shared_bench_test.go index 13c3f17e86c..1d355c9cea2 100644 --- a/db/test/domain_shared_bench_test.go +++ b/db/test/domain_shared_bench_test.go @@ -60,6 +60,15 @@ func testDbAndAggregatorBench(b *testing.B, aggStep uint64) (kv.TemporalRwDB, *s return db, db.(state.HasAgg).Agg().(*state.Aggregator) } +func newSharedDomainsBench(b *testing.B, db kv.TemporalRoDB, tx kv.TemporalTx) *execctx.SharedDomains { + b.Helper() + domains, err := execctx.NewSharedDomains(b.Context(), tx, log.New()) + require.NoError(b, err) + domains.EnableParaTrieDB(db) + require.Equal(b, execctx.PickTrieVariant(), domains.GetCommitmentCtx().Trie().Variant()) + return domains +} + func composite(k, k2 []byte) []byte { return append(bytes.Clone(k), k2...) } @@ -73,8 +82,7 @@ func Benchmark_SharedDomains_GetLatest(t *testing.B) { require.NoError(t, err) defer rwTx.Rollback() - domains, err := execctx.NewSharedDomains(t.Context(), rwTx, log.New()) - require.NoError(t, err) + domains := newSharedDomainsBench(t, db, rwTx) defer domains.Close() maxTx := stepSize * 258 @@ -156,8 +164,7 @@ func BenchmarkSharedDomains_ComputeCommitment(b *testing.B) { require.NoError(b, err) defer rwTx.Rollback() - domains, err := execctx.NewSharedDomains(b.Context(), rwTx, log.New()) - require.NoError(b, err) + domains := newSharedDomainsBench(b, db, rwTx) defer domains.Close() maxTx := stepSize * 4 @@ -338,8 +345,7 @@ func BenchmarkPruneSmallBatches(b *testing.B) { require.NoError(b, err) defer rwTx.Rollback() - domains, err := execctx.NewSharedDomains(ctx, rwTx, log.New()) - require.NoError(b, err) + domains := newSharedDomainsBench(b, db, rwTx) usedKeys := make(map[string]struct{}, keysCount*maxTx) for txNum := uint64(1); txNum <= maxTx; txNum++ { diff --git a/db/test/lifecycle_bench_test.go b/db/test/lifecycle_bench_test.go index 20f27f1002c..e58cd76ffa9 100644 --- a/db/test/lifecycle_bench_test.go +++ b/db/test/lifecycle_bench_test.go @@ -26,7 +26,6 @@ import ( "github.com/stretchr/testify/require" "github.com/erigontech/erigon/common/length" - "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/state" "github.com/erigontech/erigon/db/state/execctx" @@ -171,8 +170,7 @@ func runLifecycle(b *testing.B, cfg lifecycleConfig) (*lifecycleTimings, kv.Temp require.NoError(b, err) defer rwTx.Rollback() - domains, err := execctx.NewSharedDomains(ctx, rwTx, log.New()) - require.NoError(b, err) + domains := newSharedDomainsBench(b, db, rwTx) defer domains.Close() rnd := newRnd(42) @@ -338,8 +336,7 @@ func BenchmarkLifecycle_PhaseIsolation(b *testing.B) { require.NoError(b, err) defer rwTx.Rollback() - domains, err := execctx.NewSharedDomains(ctx, rwTx, log.New()) - require.NoError(b, err) + domains := newSharedDomainsBench(b, db, rwTx) defer domains.Close() rnd := newRnd(42) @@ -371,8 +368,7 @@ func BenchmarkLifecycle_PhaseIsolation(b *testing.B) { require.NoError(b, err) defer rwTx.Rollback() - domains, err := execctx.NewSharedDomains(ctx, rwTx, log.New()) - require.NoError(b, err) + domains := newSharedDomainsBench(b, db, rwTx) defer domains.Close() rnd := newRnd(42) @@ -413,8 +409,7 @@ func BenchmarkLifecycle_PhaseIsolation(b *testing.B) { require.NoError(b, err) defer rwTx.Rollback() - domains, err := execctx.NewSharedDomains(ctx, rwTx, log.New()) - require.NoError(b, err) + domains := newSharedDomainsBench(b, db, rwTx) defer domains.Close() txNum := initAccounts(b, domains, rwTx, keyGen) @@ -446,8 +441,7 @@ func BenchmarkLifecycle_PhaseIsolation(b *testing.B) { require.NoError(b, err) defer rwTx.Rollback() - domains, err := execctx.NewSharedDomains(ctx, rwTx, log.New()) - require.NoError(b, err) + domains := newSharedDomainsBench(b, db, rwTx) defer domains.Close() rnd := newRnd(42) diff --git a/tools/eest-spec-shards.yml b/tools/eest-spec-shards.yml index b2519efe961..9f7ce679b5d 100644 --- a/tools/eest-spec-shards.yml +++ b/tools/eest-spec-shards.yml @@ -18,7 +18,7 @@ # explaining why and a tracking issue. # expected-tests (optional) — exact result count. Used by legacy shards # to guard the coverage moved out of Go tests. -# commitment-parallel (optional, default false) — pins ERIGON_COMMITMENT_PARALLEL. +# commitment-parallel (required) — pins ERIGON_COMMITMENT_PARALLEL. # Execution is parallel in every shard # (dbg.Exec3Parallel defaults true); the # -sequential/-parallel suffix selects @@ -42,41 +42,54 @@ - shard: statetests-stable workers: 12 max-allowed-failures: 0 + commitment-parallel: false - shard: statetests-devnet workers: 12 max-allowed-failures: 0 + commitment-parallel: false - shard: statetests-legacy workers: 12 max-allowed-failures: 0 + commitment-parallel: false expected-tests: 108593 - shard: rlptests-legacy-race workers: 12 max-allowed-failures: 0 + commitment-parallel: false expected-tests: 55 no-ramdisk: true - shard: transactiontests-legacy-race workers: 12 max-allowed-failures: 0 + commitment-parallel: false expected-tests: 212 no-ramdisk: true - shard: difficultytests-legacy-race workers: 12 max-allowed-failures: 0 + commitment-parallel: false expected-tests: 18598 no-ramdisk: true - shard: blocktests-stable-sequential workers: 12 max-allowed-failures: 0 + commitment-parallel: false - shard: blocktests-stable-parallel workers: 12 max-allowed-failures: 0 commitment-parallel: true -- shard: blocktests-devnet +- shard: blocktests-devnet-sequential workers: 12 max-allowed-failures: 0 + commitment-parallel: false +- shard: blocktests-devnet-parallel + workers: 12 + max-allowed-failures: 0 + commitment-parallel: true - shard: blocktests-legacy-consensus-sequential workers: 12 max-allowed-failures: 0 + commitment-parallel: false expected-tests: 1142 exclude: - '/\.meta/' @@ -90,6 +103,7 @@ - shard: blocktests-legacy-consensus-race-sequential workers: 12 max-allowed-failures: 0 + commitment-parallel: false expected-tests: 1142 exclude: - '/\.meta/' @@ -104,6 +118,7 @@ workers: 12 # PoW fork-choice and total-difficulty support: https://github.com/erigontech/erigon/issues/22061 max-allowed-failures: 24 + commitment-parallel: false expected-tests: 32615 exclude: - '/\.meta/' @@ -119,6 +134,7 @@ workers: 12 # PoW fork-choice and total-difficulty support: https://github.com/erigontech/erigon/issues/22061 max-allowed-failures: 5 + commitment-parallel: false expected-tests: 10807 run: '_Constantinople$' exclude: @@ -136,6 +152,7 @@ workers: 12 # PoW fork-choice and total-difficulty support: https://github.com/erigontech/erigon/issues/22061 max-allowed-failures: 5 + commitment-parallel: false expected-tests: 10802 run: '_ConstantinopleFix$' exclude: @@ -152,6 +169,7 @@ - shard: blocktests-legacy-constantinople-race-byzantium-sequential workers: 8 max-allowed-failures: 0 + commitment-parallel: false expected-tests: 5000 run: '_Byzantium$' exclude: @@ -168,6 +186,7 @@ workers: 12 # PoW fork-choice and total-difficulty support: https://github.com/erigontech/erigon/issues/22061 max-allowed-failures: 14 + commitment-parallel: false expected-tests: 6006 exclude: - '/\.meta/' @@ -187,6 +206,7 @@ workers: 12 # PoW fork-choice and total-difficulty support: https://github.com/erigontech/erigon/issues/22061 max-allowed-failures: 26 + commitment-parallel: false expected-tests: 111983 exclude: - '/\.meta/' @@ -202,6 +222,7 @@ workers: 12 # PoW fork-choice and total-difficulty support: https://github.com/erigontech/erigon/issues/22061 max-allowed-failures: 7 + commitment-parallel: false expected-tests: 14026 run: '(_Berlin$|fork_Berlin-)' exclude: @@ -218,6 +239,7 @@ - shard: blocktests-legacy-cancun-race-shanghai-sequential workers: 12 max-allowed-failures: 0 + commitment-parallel: false expected-tests: 20689 run: '(_Shanghai$|fork_Shanghai-)' exclude: @@ -233,6 +255,7 @@ - shard: blocktests-legacy-cancun-race-cancun-sequential workers: 12 max-allowed-failures: 0 + commitment-parallel: false expected-tests: 21849 run: '(_Cancun$|fork_Cancun-)' exclude: @@ -249,6 +272,7 @@ workers: 12 # PoW fork-choice and total-difficulty support: https://github.com/erigontech/erigon/issues/22061 max-allowed-failures: 7 + commitment-parallel: false expected-tests: 20337 run: '(_London$|fork_London-)' exclude: @@ -265,6 +289,7 @@ - shard: blocktests-legacy-cancun-race-paris-sequential workers: 12 max-allowed-failures: 0 + commitment-parallel: false expected-tests: 20369 run: '(_Paris$|fork_Paris-)' exclude: @@ -281,6 +306,7 @@ workers: 12 # PoW fork-choice and total-difficulty support: https://github.com/erigontech/erigon/issues/22061 max-allowed-failures: 12 + commitment-parallel: false expected-tests: 14713 exclude: - '/\.meta/' @@ -297,6 +323,7 @@ - shard: enginextests-stable-sequential workers: 8 max-allowed-failures: 0 + commitment-parallel: false - shard: enginextests-stable-parallel workers: 8 max-allowed-failures: 0 @@ -304,6 +331,7 @@ - shard: enginextests-benchmark-1m-sequential workers: 1 max-allowed-failures: 0 + commitment-parallel: false no-ramdisk: true - shard: enginextests-benchmark-1m-parallel workers: 1 @@ -313,6 +341,7 @@ - shard: enginextests-benchmark-5m-sequential workers: 1 max-allowed-failures: 0 + commitment-parallel: false no-ramdisk: true - shard: enginextests-benchmark-5m-parallel workers: 1 @@ -322,6 +351,7 @@ - shard: enginextests-benchmark-10m-sequential workers: 1 max-allowed-failures: 0 + commitment-parallel: false no-ramdisk: true - shard: enginextests-benchmark-10m-parallel workers: 1 @@ -331,6 +361,7 @@ - shard: enginextests-benchmark-30m-sequential workers: 1 max-allowed-failures: 0 + commitment-parallel: false no-ramdisk: true - shard: enginextests-benchmark-30m-parallel workers: 1 @@ -340,6 +371,7 @@ - shard: enginextests-benchmark-60m-sequential workers: 1 max-allowed-failures: 0 + commitment-parallel: false no-ramdisk: true - shard: enginextests-benchmark-60m-parallel workers: 1 @@ -349,6 +381,7 @@ - shard: enginextests-benchmark-100m-sequential workers: 1 max-allowed-failures: 0 + commitment-parallel: false no-ramdisk: true - shard: enginextests-benchmark-100m-parallel workers: 1 @@ -358,6 +391,7 @@ - shard: enginextests-benchmark-150m-sequential workers: 1 max-allowed-failures: 0 + commitment-parallel: false no-ramdisk: true - shard: enginextests-benchmark-150m-parallel workers: 1 @@ -367,21 +401,26 @@ - shard: zkevm-witness workers: 8 max-allowed-failures: 0 + commitment-parallel: false - shard: blocktests-stable-race-pre-cancun-sequential workers: 12 max-allowed-failures: 0 + commitment-parallel: false run: 'fork_(Frontier|Homestead|TangerineWhistle|SpuriousDragon|Byzantium|ConstantinopleFix|Istanbul|Berlin|London|Paris|Shanghai)' - shard: blocktests-stable-race-cancun-sequential workers: 12 max-allowed-failures: 0 + commitment-parallel: false run: 'fork_Cancun' - shard: blocktests-stable-race-prague-sequential workers: 12 max-allowed-failures: 0 + commitment-parallel: false run: 'fork_Prague' - shard: blocktests-stable-race-osaka-sequential workers: 12 max-allowed-failures: 0 + commitment-parallel: false run: 'fork_(Osaka|BPO)' - shard: blocktests-stable-race-pre-cancun-parallel workers: 12 @@ -403,10 +442,17 @@ max-allowed-failures: 0 commitment-parallel: true run: 'fork_(Osaka|BPO)' -- shard: blocktests-devnet-race-amsterdam +- shard: blocktests-devnet-race-amsterdam-sequential workers: 12 max-allowed-failures: 0 + commitment-parallel: false + run: 'fork_Amsterdam' +- shard: blocktests-devnet-race-amsterdam-parallel + workers: 12 + max-allowed-failures: 0 + commitment-parallel: true run: 'fork_Amsterdam' - shard: zkevm-witness-race workers: 8 max-allowed-failures: 0 + commitment-parallel: false diff --git a/tools/run-eest-spec-test.sh b/tools/run-eest-spec-test.sh index 549eeb0db8d..253614181c3 100755 --- a/tools/run-eest-spec-test.sh +++ b/tools/run-eest-spec-test.sh @@ -12,7 +12,7 @@ # transactiontests-legacy-race complete legacy transaction tests # difficultytests-legacy-race complete legacy difficulty tests # blocktests-stable-sequential blockchain tests vs. eest_stable -# blocktests-devnet blockchain tests vs. eest_devnet +# blocktests-devnet-{sequential,parallel} blockchain tests vs. eest_devnet # blocktests-legacy-consensus-{sequential,parallel} # Hive consensus suite # blocktests-legacy-consensus-race-{sequential,parallel} @@ -44,7 +44,8 @@ # export EVM_BIN to the race-built # binary; otherwise -race # detection doesn't fire. -# blocktests-devnet-race-amsterdam race-detector variant filtered +# blocktests-devnet-race-amsterdam-{sequential,parallel} +# race-detector variants filtered # to the Amsterdam fork only. # zkevm-witness zkevm execution-witness conformance # (eest_zkevm corpus) via the zkevmtest @@ -117,7 +118,12 @@ base=$(fixture_base "${fixture_sets[0]}") # adding a shard / tweaking a budget / changing a fork filter is a one-file edit. # yq converts YAML→JSON so it can be queried with jq. manifest=tools/eest-spec-shards.yml -shard_row=$(yq -o=json '.' "$manifest" | jq -r --arg s "$shard" '.[] | select(.shard == $s) | "\(.workers)\t\(."max-allowed-failures")\t\(."commitment-parallel" // false)\t\(."no-ramdisk" // false)\t\(."expected-tests" // 0)\t\(.run // "")"') +shard_row=$(yq -o=json '.' "$manifest" | jq -r --arg s "$shard" ' + .[] | select(.shard == $s) + | if (."commitment-parallel" | type) != "boolean" + then error("shard \($s) must define boolean commitment-parallel") + else "\(.workers)\t\(."max-allowed-failures")\t\(."commitment-parallel")\t\(."no-ramdisk" // false)\t\(."expected-tests" // 0)\t\(.run // "")" + end') if [[ -z "$shard_row" ]]; then echo "shard $shard not found in $manifest" >&2 exit 2 From a06042325f948c60eb406a8558ce09f0f2adc8f1 Mon Sep 17 00:00:00 2001 From: taratorio <94537774+taratorio@users.noreply.github.com> Date: Fri, 31 Jul 2026 15:45:14 +0200 Subject: [PATCH 8/9] cmd/evm, ci: run statetests in both commitment modes --- cmd/evm/staterunner.go | 12 +++++++- cmd/evm/staterunner_test.go | 57 +++++++++++++++++++++++++++++++++++++ tools/eest-spec-shards.yml | 19 +++++++++++-- tools/run-eest-spec-test.sh | 8 +++--- 4 files changed, 88 insertions(+), 8 deletions(-) create mode 100644 cmd/evm/staterunner_test.go diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index 0bd784dd5cb..ec0022c0f6a 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -33,6 +33,7 @@ import ( "github.com/erigontech/erigon/common/dir" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/datadir" + "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/temporal/temporaltest" "github.com/erigontech/erigon/db/state/execctx" "github.com/erigontech/erigon/execution/tests/testutil" @@ -61,6 +62,15 @@ var stateTestCommand = cli.Command{ }, } +func newStateTestSharedDomains(db kv.TemporalRoDB, tx kv.TemporalTx) (*execctx.SharedDomains, error) { + sd, err := execctx.NewSharedDomains(context.Background(), tx, log.New()) + if err != nil { + return nil, err + } + sd.EnableParaTrieDB(db) + return sd, nil +} + func stateTestCmd(_ context.Context, ctx *cli.Command) error { machineFriendlyOutput := ctx.Bool(MachineFlag.Name) if machineFriendlyOutput { @@ -224,7 +234,7 @@ func runStateTest(ctx *cli.Command, cfg vm.Config, fname string, filter testFilt defer tx.Rollback() // Per-subtest SD: closed without Flush so its writes never enter the branch cache. - sd, err := execctx.NewSharedDomains(context.Background(), tx, log.New()) + sd, err := newStateTestSharedDomains(db, tx) if err != nil { result.Pass, result.Error = false, err.Error() return diff --git a/cmd/evm/staterunner_test.go b/cmd/evm/staterunner_test.go new file mode 100644 index 00000000000..f91ebb9d57e --- /dev/null +++ b/cmd/evm/staterunner_test.go @@ -0,0 +1,57 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/erigontech/erigon/db/kv/temporal/temporaltest" + "github.com/erigontech/erigon/db/state/statecfg" + "github.com/erigontech/erigon/execution/commitment" +) + +func TestNewStateTestSharedDomainsUsesSelectedCommitment(t *testing.T) { + originalParallel := statecfg.ExperimentalParallelCommitment + originalStreaming := statecfg.ExperimentalStreamingCommitment + t.Cleanup(func() { + statecfg.ExperimentalParallelCommitment = originalParallel + statecfg.ExperimentalStreamingCommitment = originalStreaming + }) + + for _, tc := range []struct { + name string + parallel bool + variant commitment.TrieVariant + }{ + {name: "serial", variant: commitment.VariantHexPatriciaTrie}, + {name: "parallel", parallel: true, variant: commitment.VariantParallelHexPatricia}, + } { + t.Run(tc.name, func(t *testing.T) { + statecfg.ExperimentalParallelCommitment = tc.parallel + statecfg.ExperimentalStreamingCommitment = false + + db, tx := temporaltest.NewTestTx(t) + sd, err := newStateTestSharedDomains(db, tx) + require.NoError(t, err) + t.Cleanup(sd.Close) + + require.Equal(t, tc.variant, sd.GetCommitmentCtx().Trie().Variant()) + }) + } +} diff --git a/tools/eest-spec-shards.yml b/tools/eest-spec-shards.yml index 9f7ce679b5d..6ab05a46b06 100644 --- a/tools/eest-spec-shards.yml +++ b/tools/eest-spec-shards.yml @@ -39,19 +39,32 @@ # -sequential/-parallel variants of a fork # carry the same regex. -- shard: statetests-stable +- shard: statetests-stable-sequential workers: 12 max-allowed-failures: 0 commitment-parallel: false -- shard: statetests-devnet +- shard: statetests-stable-parallel + workers: 12 + max-allowed-failures: 0 + commitment-parallel: true +- shard: statetests-devnet-sequential workers: 12 max-allowed-failures: 0 commitment-parallel: false -- shard: statetests-legacy +- shard: statetests-devnet-parallel + workers: 12 + max-allowed-failures: 0 + commitment-parallel: true +- shard: statetests-legacy-sequential workers: 12 max-allowed-failures: 0 commitment-parallel: false expected-tests: 108593 +- shard: statetests-legacy-parallel + workers: 12 + max-allowed-failures: 0 + commitment-parallel: true + expected-tests: 108593 - shard: rlptests-legacy-race workers: 12 max-allowed-failures: 0 diff --git a/tools/run-eest-spec-test.sh b/tools/run-eest-spec-test.sh index 253614181c3..b4bc7264604 100755 --- a/tools/run-eest-spec-test.sh +++ b/tools/run-eest-spec-test.sh @@ -5,9 +5,9 @@ # # Where is one of: # -# statetests-stable state tests vs. eest_stable -# statetests-devnet state tests vs. eest_devnet -# statetests-legacy complete legacy Cancun state tests +# statetests-stable-{sequential,parallel} state tests vs. eest_stable +# statetests-devnet-{sequential,parallel} state tests vs. eest_devnet +# statetests-legacy-{sequential,parallel} complete legacy Cancun state tests # rlptests-legacy-race complete legacy RLP tests # transactiontests-legacy-race complete legacy transaction tests # difficultytests-legacy-race complete legacy difficulty tests @@ -95,7 +95,7 @@ case "$shard" in *-stable*) fixture_sets=(eest_stable) ;; *-devnet*) fixture_sets=(eest_devnet) ;; *-benchmark*) fixture_sets=(eest_benchmark) ;; - statetests-legacy) fixture_sets=(legacy_cancun) ;; + statetests-legacy-*) fixture_sets=(legacy_cancun) ;; rlptests-legacy-* | \ transactiontests-legacy-* | \ difficultytests-legacy-*) fixture_sets=(legacy_tests) ;; From 0baa31151b9b49f9e5f546466d21dceb4341ef2e Mon Sep 17 00:00:00 2001 From: taratorio <94537774+taratorio@users.noreply.github.com> Date: Fri, 31 Jul 2026 16:43:59 +0200 Subject: [PATCH 9/9] ci: update EEST test skill targets --- .claude/skills/erigon-ci/SKILL.md | 2 +- .claude/skills/erigon-implement-eip/SKILL.md | 16 +++++++--------- .claude/skills/erigon-test-all/SKILL.md | 17 +++++++++++------ .claude/skills/erigon-test-race/SKILL.md | 4 ++-- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/.claude/skills/erigon-ci/SKILL.md b/.claude/skills/erigon-ci/SKILL.md index e570d9a7fdc..2835d4dfeda 100644 --- a/.claude/skills/erigon-ci/SKILL.md +++ b/.claude/skills/erigon-ci/SKILL.md @@ -22,7 +22,7 @@ Each test group has its own dedicated skill for drill-down on failures. Use thos | unit | `erigon-test-unit` | `make test-short` | ~5 min | Pre-push gate | | all | `erigon-test-all` | `GOGC=80 make test-all` | ~30 min | Before PR review | | race | `erigon-test-race` | `make test-all-race` | ~60 min | Concurrency changes | -| eest-spec | *(inline)* | `make eest-spec--[-{sequential,parallel}]` | varies | EEST state/blockchain/engine-x changes (most shards split into `-sequential` / `-parallel` pairs that pin `ERIGON_EXEC3_PARALLEL`; see `tools/eest-spec-shards.yml`) | +| eest-spec | *(inline)* | `make eest-spec-` | varies | EEST state/blockchain/engine-x changes (paired shards pin `ERIGON_COMMITMENT_PARALLEL`; execution remains parallel; see `tools/eest-spec-shards.yml`) | | caplin spec | *(inline)* | `cd cl/spectest && make tests && make mainnet` | ~15 min | CL/consensus changes | | hive | `erigon-test-hive` | `make test-hive` | ~20 min | EL/CL interop changes | | rpc | `erigon-test-rpc` | *(requires synced DB)* | ~10 min | RPC API changes | diff --git a/.claude/skills/erigon-implement-eip/SKILL.md b/.claude/skills/erigon-implement-eip/SKILL.md index 12a677469d9..ae52627f010 100644 --- a/.claude/skills/erigon-implement-eip/SKILL.md +++ b/.claude/skills/erigon-implement-eip/SKILL.md @@ -115,15 +115,13 @@ Run local tests using the `/erigon-test-all` skill. Analyse and fix any failures The most important tests when implementing a new EIP for the EL are the EEST spec test shards, exercised by the `cmd/evm` runners (`statetest`, `blocktest`, `enginextest`) via the Makefile targets: -- `make eest-spec-statetests-stable` / `…-devnet` — state-tests against the stable/devnet EEST fixtures -- `make eest-spec-blocktests-stable-sequential` / `…-devnet` — blockchain-tests against the stable/devnet EEST fixtures. The devnet shard always runs under `ERIGON_EXEC3_PARALLEL=true` (the in-development hardfork requires it); the `…-sequential` shard pins `ERIGON_EXEC3_PARALLEL=false`. -- `make eest-spec-blocktests-stable-parallel` — same fixtures as `…-stable-sequential` but with `ERIGON_EXEC3_PARALLEL=true`; useful for catching parallel-only regressions on stable fixtures. -- `make eest-spec-enginextests-stable-sequential` — engine-x tests against the stable EEST fixtures with `ERIGON_EXEC3_PARALLEL=false`. No devnet variant: the devnet tarball doesn't yet ship `blockchain_tests_engine_x/`. -- `make eest-spec-enginextests-stable-parallel` — same fixtures as `…-stable-sequential` but with `ERIGON_EXEC3_PARALLEL=true`; useful for catching parallel-only regressions on engine-x stable fixtures. -- `make eest-spec-enginextests-benchmark-{1m,5m,10m,30m,60m,100m,150m}-{sequential,parallel}` — engine-x tests against the per-gas-target benchmark fixtures, with `--time` per-test stats. Each gas target has a `-sequential` (`ERIGON_EXEC3_PARALLEL=false`) and `-parallel` (`ERIGON_EXEC3_PARALLEL=true`) variant. -- `make eest-spec-blocktests-stable-race-{pre-cancun,cancun,prague,osaka}-{sequential,parallel}` and `make eest-spec-blocktests-devnet-race-amsterdam` — race-detector variants split by fork. Each stable-race sub-shard has a `-sequential` / `-parallel` pair; the `-parallel` siblings exercise parallel exec3 under the race detector. The `blocktests-devnet-race-amsterdam` shard is always parallel (matches the non-race devnet behaviour). - -The shard list / failure budgets / `exec3-parallel` flags are defined in `tools/eest-spec-shards.yml` (single source of truth shared with the CI workflow and the local runner script). See `EEST_SPEC_SHARDS` / `EEST_SPEC_RACE_SHARDS` in the root `Makefile` for the partition into non-race vs race targets. +- `make eest-spec-statetests-{stable,devnet}-{sequential,parallel}` — state tests against the stable/devnet EEST fixtures in both commitment modes. +- `make eest-spec-blocktests-{stable,devnet}-{sequential,parallel}` — blockchain tests against the stable/devnet EEST fixtures in both commitment modes. +- `make eest-spec-enginextests-stable-{sequential,parallel}` — engine-x tests against the stable EEST fixtures in both commitment modes. No devnet variant exists because the devnet tarball does not yet ship `blockchain_tests_engine_x/`. +- `make eest-spec-enginextests-benchmark-{1m,5m,10m,30m,60m,100m,150m}-{sequential,parallel}` — engine-x tests against the per-gas-target benchmark fixtures, with `--time` per-test stats and both commitment modes. +- `make eest-spec-blocktests-stable-race-{pre-cancun,cancun,prague,osaka}-{sequential,parallel}` and `make eest-spec-blocktests-devnet-race-amsterdam-{sequential,parallel}` — race-detector variants split by fork and commitment mode. + +The shard list, failure budgets, and `commitment-parallel` flags are defined in `tools/eest-spec-shards.yml` (single source of truth shared with the CI workflow and the local runner script). Execution remains parallel in every paired shard. See `EEST_SPEC_SHARDS` / `EEST_SPEC_RACE_SHARDS` in the root `Makefile` for the partition into non-race vs race targets. **Pitfall: stale `evm` / `evm.race` binary.** When iterating on an EIP implementation, always invoke shards via `make eest-spec-` rather than `bash tools/run-eest-spec-test.sh ` — the make target lists `evm` (or `evm.race`) as a prereq and `go build` is cache-aware, so a fresh binary is built before each run. The script invoked directly **bypasses** that rebuild, so the runners exercise whatever `build/bin/evm{,.race}` happens to be on disk against current fixtures — silently inflating failures (e.g. devnet shards "regressing" by thousands of tests) or hiding regressions when comparing budgets before/after a change. diff --git a/.claude/skills/erigon-test-all/SKILL.md b/.claude/skills/erigon-test-all/SKILL.md index 0676ee522ce..2abc8e1b1a2 100644 --- a/.claude/skills/erigon-test-all/SKILL.md +++ b/.claude/skills/erigon-test-all/SKILL.md @@ -14,14 +14,18 @@ Runs the complete test suite with 60-minute timeout and coverage output. Takes ~ To exercise the EEST suites locally, see `erigon-eest-spec` (or run a specific shard directly): ```bash -make eest-spec-statetests-stable # state tests vs eest_stable fixtures +make eest-spec-statetests-stable-{sequential,parallel} + # state tests vs eest_stable fixtures make eest-spec-blocktests-stable-sequential # blockchain tests vs eest_stable fixtures (serial commitment) make eest-spec-blocktests-stable-parallel # same, but with parallel commitment make eest-spec-enginextests-stable-sequential # engine-x tests vs eest_stable (serial commitment) make eest-spec-enginextests-stable-parallel # same, but with parallel commitment -make eest-spec-statetests-devnet # …vs eest_devnet fixtures -make eest-spec-blocktests-devnet # devnet blocktests (serial commitment) -make eest-spec-statetests-legacy # pinned legacy Cancun state-test archive +make eest-spec-statetests-devnet-{sequential,parallel} + # state tests vs eest_devnet fixtures +make eest-spec-blocktests-devnet-{sequential,parallel} + # devnet blocktests in both commitment modes +make eest-spec-statetests-legacy-{sequential,parallel} + # pinned legacy Cancun state-test archive make eest-spec-rlptests-legacy-race # complete pinned legacy RLP suite make eest-spec-transactiontests-legacy-race # complete pinned legacy transaction suite make eest-spec-difficultytests-legacy-race # complete pinned legacy difficulty suite @@ -50,8 +54,9 @@ make eest-spec-enginextests-benchmark-1m-sequential make eest-spec-blocktests-stable-race-cancun-sequential # race-detector variant, sharded per fork: # -pre-cancun/-cancun/-prague/-osaka, plus - # eest-spec-blocktests-devnet-race-amsterdam. - # Each stable-race sub-shard has a + # eest-spec-blocktests-devnet-race-amsterdam- + # {sequential,parallel}. Each stable-race + # and devnet-race sub-shard has a # "-sequential" / "-parallel" pair # (e.g. ...-race-cancun-{sequential,parallel}) ``` diff --git a/.claude/skills/erigon-test-race/SKILL.md b/.claude/skills/erigon-test-race/SKILL.md index 3e436481960..5acf47a2d21 100644 --- a/.claude/skills/erigon-test-race/SKILL.md +++ b/.claude/skills/erigon-test-race/SKILL.md @@ -16,13 +16,13 @@ Use the dedicated EEST race shards: ```bash make eest-spec-{rlptests,transactiontests,difficultytests}-legacy-race make eest-spec-blocktests-stable-race-{pre-cancun,cancun,prague,osaka}-{sequential,parallel} -make eest-spec-blocktests-devnet-race-amsterdam +make eest-spec-blocktests-devnet-race-amsterdam-{sequential,parallel} make eest-spec-blocktests-legacy-consensus-race-{sequential,parallel} make eest-spec-blocktests-legacy-constantinople-race-{constantinople,constantinople-fix,other-forks}-{sequential,parallel} make eest-spec-blocktests-legacy-cancun-race-{berlin,shanghai,cancun,london,paris,other-forks}-{sequential,parallel} ``` -These targets build a race-instrumented `evm.race` binary automatically (see `EEST_SPEC_RACE_SHARDS` in the root `Makefile`). The stable and legacy `-sequential` / `-parallel` pairs pin both commitment modes; execution remains parallel in every pair. The unsuffixed devnet race shard uses serial commitment. For the consensus spec suite or other Go packages, pass `GOFLAGS='-race'` or invoke `go test -race` against the relevant package directly. +These targets build a race-instrumented `evm.race` binary automatically (see `EEST_SPEC_RACE_SHARDS` in the root `Makefile`). The stable, devnet, and legacy blocktest `-sequential` / `-parallel` pairs pin both commitment modes; execution remains parallel in every pair. For the consensus spec suite or other Go packages, pass `GOFLAGS='-race'` or invoke `go test -race` against the relevant package directly. **Pitfall: stale `evm.race` binary.** `make eest-spec-` lists `evm.race` as a prereq and `go build` is cache-aware, so a stale binary gets rebuilt. Calling `bash tools/run-eest-spec-test.sh ` directly with `EVM_BIN=build/bin/evm.race` **bypasses** the rebuild and silently runs an old race-instrumented binary against current fixtures — race reports against code that no longer exists, missed races against code that does. After pulling or switching branches: `rm -f build/bin/evm.race && make evm.race` before re-running.