execution/commitment: commitment metrics as counters, round duration and branch IO bytes

cmd/prometheus/dashboards/erigon_internals.json

@@ -5144,7 +5144,7 @@
           "targets": [
             {
               "editorMode": "code",
-              "expr": "commit_key_rate{instance=~\"$instance\"}",
+              "expr": "rate(commitment_keys_total{instance=~\"$instance\"}[$__rate_interval])",
               "legendFormat": "total keys",
               "range": true,
               "refId": "A",
@@ -5155,7 +5155,7 @@
             },
             {
               "editorMode": "code",
-              "expr": "commit_account_key_rate{instance=~\"$instance\"}",
+              "expr": "rate(commitment_key_traversals_total{kind=\"address\",instance=~\"$instance\"}[$__rate_interval])",
               "legendFormat": "account keys",
               "range": true,
               "refId": "B",
@@ -5166,7 +5166,7 @@
             },
             {
               "editorMode": "code",
-              "expr": "commit_storage_key_rate{instance=~\"$instance\"}",
+              "expr": "rate(commitment_key_traversals_total{kind=\"storage\",instance=~\"$instance\"}[$__rate_interval])",
               "legendFormat": "storage keys",
               "range": true,
               "refId": "C",
@@ -5265,7 +5265,7 @@
           "targets": [
             {
               "editorMode": "code",
-              "expr": "commit_fold_rate{instance=~\"$instance\"}",
+              "expr": "rate(commitment_folds_total{instance=~\"$instance\"}[$__rate_interval])",
               "legendFormat": "fold",
               "range": true,
               "refId": "A",
@@ -5276,7 +5276,7 @@
             },
             {
               "editorMode": "code",
-              "expr": "commit_unfold_rate{instance=~\"$instance\"}",
+              "expr": "rate(commitment_unfolds_total{instance=~\"$instance\"}[$__rate_interval])",
               "legendFormat": "unfold",
               "range": true,
               "refId": "B",
@@ -5375,7 +5375,7 @@
           "targets": [
             {
               "editorMode": "code",
-              "expr": "commit_branch_read_rate{instance=~\"$instance\"}",
+              "expr": "rate(commitment_reads_total{kind=\"branch\",instance=~\"$instance\"}[$__rate_interval])",
               "legendFormat": "branch reads",
               "range": true,
               "refId": "A",
@@ -5386,7 +5386,7 @@
             },
             {
               "editorMode": "code",
-              "expr": "commit_branch_write_rate{instance=~\"$instance\"}",
+              "expr": "rate(commitment_branch_writes_total{instance=~\"$instance\"}[$__rate_interval])",
               "legendFormat": "branch writes",
               "range": true,
               "refId": "B",
@@ -5397,7 +5397,7 @@
             },
             {
               "editorMode": "code",
-              "expr": "commit_read_rate{instance=~\"$instance\"}",
+              "expr": "sum without (kind) (rate(commitment_reads_total{instance=~\"$instance\"}[$__rate_interval]))",
               "legendFormat": "total reads",
               "range": true,
               "refId": "C",
@@ -5408,7 +5408,7 @@
             },
             {
               "editorMode": "code",
-              "expr": "commit_account_read_rate{instance=~\"$instance\"}",
+              "expr": "rate(commitment_reads_total{kind=\"account\",instance=~\"$instance\"}[$__rate_interval])",
               "legendFormat": "account reads",
               "range": true,
               "refId": "D",
@@ -5419,7 +5419,7 @@
             },
             {
               "editorMode": "code",
-              "expr": "commit_storage_read_rate{instance=~\"$instance\"}",
+              "expr": "rate(commitment_reads_total{kind=\"storage\",instance=~\"$instance\"}[$__rate_interval])",
               "legendFormat": "storage reads",
               "range": true,
               "refId": "E",
@@ -5535,8 +5535,8 @@
           "targets": [
             {
               "editorMode": "code",
-              "expr": "commit_block_dur{instance=~\"$instance\"}",
-              "legendFormat": "block dur (ns)",
+              "expr": "1e9 * histogram_quantile(0.9, sum by (le, instance) (rate(commitment_round_duration_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])))",
+              "legendFormat": "block dur (ns) {{instance}}",
               "range": true,
               "refId": "A",
               "datasource": {

dashboards/erigon_custom_metrics/erigon_custom_metrics.internal.json

@@ -5997,7 +5997,7 @@
                 "uid": "grafanacloud-prom"
               },
               "editorMode": "builder",
-              "expr": "commit_block_dur{instance=\"$instance\"}",
+              "expr": "1e9 * histogram_quantile(0.9, sum by (le, instance) (rate(commitment_round_duration_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])))",
               "instant": false,
               "legendFormat": "commitment: {{instance}}",
               "range": true,
@@ -9290,7 +9290,7 @@
                 "uid": "grafanacloud-prom"
               },
               "editorMode": "code",
-              "expr": "commit_key_rate{instance=~\"$instance\"}",
+              "expr": "rate(commitment_keys_total{instance=~\"$instance\"}[$__rate_interval])",
               "legendFormat": "total keys {{instance}}",
               "range": true,
               "refId": "A"
@@ -9301,7 +9301,7 @@
                 "uid": "grafanacloud-prom"
               },
               "editorMode": "code",
-              "expr": "commit_account_key_rate{instance=~\"$instance\"}",
+              "expr": "rate(commitment_key_traversals_total{kind=\"address\",instance=~\"$instance\"}[$__rate_interval])",
               "legendFormat": "account keys {{instance}}",
               "range": true,
               "refId": "B"
@@ -9312,7 +9312,7 @@
                 "uid": "grafanacloud-prom"
               },
               "editorMode": "code",
-              "expr": "commit_storage_key_rate{instance=~\"$instance\"}",
+              "expr": "rate(commitment_key_traversals_total{kind=\"storage\",instance=~\"$instance\"}[$__rate_interval])",
               "legendFormat": "storage keys {{instance}}",
               "range": true,
               "refId": "C"
@@ -9417,7 +9417,7 @@
                 "uid": "grafanacloud-prom"
               },
               "editorMode": "code",
-              "expr": "commit_fold_rate{instance=~\"$instance\"}",
+              "expr": "rate(commitment_folds_total{instance=~\"$instance\"}[$__rate_interval])",
               "legendFormat": "fold {{instance}}",
               "range": true,
               "refId": "A"
@@ -9428,7 +9428,7 @@
                 "uid": "grafanacloud-prom"
               },
               "editorMode": "code",
-              "expr": "commit_unfold_rate{instance=~\"$instance\"}",
+              "expr": "rate(commitment_unfolds_total{instance=~\"$instance\"}[$__rate_interval])",
               "legendFormat": "unfold {{instance}}",
               "range": true,
               "refId": "B"
@@ -9533,7 +9533,7 @@
                 "uid": "grafanacloud-prom"
               },
               "editorMode": "code",
-              "expr": "commit_branch_read_rate{instance=~\"$instance\"}",
+              "expr": "rate(commitment_reads_total{kind=\"branch\",instance=~\"$instance\"}[$__rate_interval])",
               "legendFormat": "reads {{instance}}",
               "range": true,
               "refId": "A"
@@ -9544,7 +9544,7 @@
                 "uid": "grafanacloud-prom"
               },
               "editorMode": "code",
-              "expr": "commit_branch_write_rate{instance=~\"$instance\"}",
+              "expr": "rate(commitment_branch_writes_total{instance=~\"$instance\"}[$__rate_interval])",
               "legendFormat": "writes {{instance}}",
               "range": true,
               "refId": "B"
@@ -9555,7 +9555,7 @@
                 "uid": "grafanacloud-prom"
               },
               "editorMode": "code",
-              "expr": "commit_read_rate{instance=~\"$instance\"}",
+              "expr": "sum without (kind) (rate(commitment_reads_total{instance=~\"$instance\"}[$__rate_interval]))",
               "legendFormat": "total reads {{instance}}",
               "range": true,
               "refId": "C"
@@ -9566,7 +9566,7 @@
                 "uid": "grafanacloud-prom"
               },
               "editorMode": "code",
-              "expr": "commit_account_read_rate{instance=~\"$instance\"}",
+              "expr": "rate(commitment_reads_total{kind=\"account\",instance=~\"$instance\"}[$__rate_interval])",
               "legendFormat": "account reads {{instance}}",
               "range": true,
               "refId": "D"
@@ -9577,7 +9577,7 @@
                 "uid": "grafanacloud-prom"
               },
               "editorMode": "code",
-              "expr": "commit_storage_read_rate{instance=~\"$instance\"}",
+              "expr": "rate(commitment_reads_total{kind=\"storage\",instance=~\"$instance\"}[$__rate_interval])",
               "legendFormat": "storage reads {{instance}}",
               "range": true,
               "refId": "E"
@@ -9699,7 +9699,7 @@
                 "uid": "grafanacloud-prom"
               },
               "editorMode": "code",
-              "expr": "commit_block_dur{instance=~\"$instance\"}",
+              "expr": "1e9 * histogram_quantile(0.9, sum by (le, instance) (rate(commitment_round_duration_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])))",
               "legendFormat": "block dur (ns) {{instance}}",
               "range": true,
               "refId": "A"

db/state/execctx/domain_shared.go

@@ -539,7 +539,7 @@ func (sd *SharedDomains) flushPendingUpdates(ctx context.Context, tx kv.Temporal
 
 	switcher, ok := sd.mem.(changesetSwitcher)
 	if !ok {
-		_, err := commitment.ApplyDeferredBranchUpdates(upd.Deferred, runtime.NumCPU(), putBranch)
+		_, err := commitment.ApplyDeferredBranchUpdates(upd.Deferred, runtime.NumCPU(), putBranch, upd.Metrics)
 		return err
 	}
 
@@ -563,7 +563,7 @@ func (sd *SharedDomains) flushPendingUpdates(ctx context.Context, tx kv.Temporal
 		// see concurrency contract on the wrappers above.
 		defer sd.SwapCommitmentDiffLocked(cs)()
 
-		if _, err := commitment.ApplyDeferredBranchUpdates(upd.Deferred, runtime.NumCPU(), putBranch); err != nil {
+		if _, err := commitment.ApplyDeferredBranchUpdates(upd.Deferred, runtime.NumCPU(), putBranch, upd.Metrics); err != nil {
 			return err
 		}
 
@@ -572,7 +572,7 @@ func (sd *SharedDomains) flushPendingUpdates(ctx context.Context, tx kv.Temporal
 	}
 
 	// No past changeset found — write into whatever is current.
-	_, err := commitment.ApplyDeferredBranchUpdates(upd.Deferred, runtime.NumCPU(), putBranch)
+	_, err := commitment.ApplyDeferredBranchUpdates(upd.Deferred, runtime.NumCPU(), putBranch, upd.Metrics)
 	return err
 }
 

execution/commitment/commitment.go

@@ -280,6 +280,10 @@
 	BlockHash common.Hash
 	TxNum     uint64
 	Deferred  []*DeferredBranchUpdate
+	// Metrics is the producing trie's, carried so the later apply still reaches
+	// that trie's log and CSV counters. The Prometheus counters do not depend on
+	// it — publishBranchWrites bills those where the write lands.
+	Metrics *Metrics
 }
 
 func (p *PendingCommitmentUpdate) Clear() {
@@ -366,26 +370,29 @@
 	numWorkers int,
 	putBranch func(prefix []byte, data []byte, prevData []byte) error,
 ) error {
-	written, err := ApplyDeferredBranchUpdates(be.deferred, numWorkers, putBranch)
-	if err != nil {
+	if _, err := ApplyDeferredBranchUpdates(be.deferred, numWorkers, putBranch, be.metrics); err != nil {
 		return err
 	}
-	if be.metrics != nil {
-		be.metrics.updateBranch.Add(uint64(written))
-	}
 	return nil
 }
 
 var workerMergerPool = sync.Pool{New: func() any { return NewHexBranchMerger(512) }}
 
-// Returns the number of updates written. putBranch must copy prefix and data rather than
-// retain them: they are pooled and reused for a later, unrelated update. prevData is
-// cloned per update and carries no such constraint.
+// ApplyDeferredBranchUpdates applies the queued branch writes and returns how many
+// were written. Writes are published to the branch-write counters as they land,
+// not against a round: the caller-owned path applies from SharedDomains after the
+// producing round has already closed, so there is no round left to bill. m, when
+// non-nil, additionally carries them into that trie's log and CSV counters.
+//
+// putBranch must copy prefix and data rather than retain them: they are pooled and
+// reused for a later, unrelated update. prevData is cloned per update and carries
+// no such constraint.
 func ApplyDeferredBranchUpdates(
 	deferred []*DeferredBranchUpdate,
 	numWorkers int,
 	putBranch func(prefix []byte, data []byte, prevData []byte) error,
-) (int, error) {
+	m *Metrics,
+) (n int, err error) {
 	if len(deferred) == 0 {
 		return 0, nil
 	}
@@ -397,20 +404,24 @@
 		merger := workerMergerPool.Get().(*BranchMerger)
 		defer workerMergerPool.Put(merger)
 
-		var written int
+		var written, bytesOut int
 		for _, upd := range deferred {
 			if err := mergeDeferredUpdate(upd, merger); err != nil {
+				publishBranchWrites(written, bytesOut, m)
 				return written, err
 			}
 			if upd.encoded == nil {
 				continue
 			}
 			if err := putBranch(capLen(upd.prefix), capLen(upd.encoded), capLen(upd.prev)); err != nil {
+				publishBranchWrites(written, bytesOut, m)
 				return written, err
 			}
 			written++
+			bytesOut += len(upd.encoded)
 		}
 		mxTrieBranchesUpdated.AddInt(written)
+		publishBranchWrites(written, bytesOut, m)
 		return written, nil
 	}
 
@@ -442,17 +453,20 @@
 		}
 	}
 
-	var written int
+	var written, bytesOut int
 	for _, upd := range deferred {
 		if upd.encoded == nil {
 			continue
 		}
 		if err := putBranch(capLen(upd.prefix), capLen(upd.encoded), capLen(upd.prev)); err != nil {
+			publishBranchWrites(written, bytesOut, m)
 			return written, err
 		}
 		written++
+		bytesOut += len(upd.encoded)
 	}
 	mxTrieBranchesUpdated.AddInt(written)
+	publishBranchWrites(written, bytesOut, m)
 	return written, nil
 }
 
@@ -500,9 +514,7 @@
 	if err := ctx.PutBranch(prefixCopy, updateCopy, prev); err != nil {
 		return err
 	}
-	if be.metrics != nil {
-		be.metrics.updateBranch.Add(1)
-	}
+	publishBranchWrites(1, len(updateCopy), be.metrics)
 	mxTrieBranchesUpdated.Inc()
 	return nil
 }
@@ -1741,7 +1753,7 @@
 }
 
 // fn must not retain hk or pk slices after returning: they're backed by reusable arena memory.
 func (t *Updates) HashSort(ctx context.Context, warmuper *Warmuper, fn func(hk, pk []byte, update *Update) error) error {
 	switch t.mode {
 	case ModeDirect:
 		cnt := len(t.keys)

execution/commitment/commitment_test.go

@@ -1105,7 +1105,7 @@ func TestApplyDeferred_CallbackSeesInputDerivedCapacity(t *testing.T) {
 					require.Equal(t, len(data), cap(data), "data carries leftover pool capacity")
 					require.Equal(t, len(prevData), cap(prevData), "prevData carries leftover pool capacity")
 					return nil
-				})
+				}, nil)
 			require.NoError(t, err)
 			require.Equal(t, tc.updates, written)
 			require.Equal(t, tc.updates, seen, "callback must run for every update")

execution/commitment/commitmentdb/commitment_context.go

@@ -601,6 +601,7 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context
 				BlockNum: blockNum,
 				TxNum:    txNum,
 				Deferred: trie.TakeDeferredUpdates(),
+				Metrics:  trie.Metrics(),
 			}
 		}
 	case *commitment.ParallelPatriciaHashed:
@@ -609,6 +610,7 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context
 				BlockNum: blockNum,
 				TxNum:    txNum,
 				Deferred: trie.TakeDeferredUpdates(),
+				Metrics:  trie.Metrics(),
 			}
 		}
 	}

execution/commitment/hex_patricia_hashed.go

@@ -233,6 +233,10 @@ func newHexPatriciaHashed() *HexPatriciaHashed {
 	return hph
 }
 
+// Metrics exposes the trie's counters so a caller applying its deferred writes
+// can carry them into this trie's log and CSV totals.
+func (hph *HexPatriciaHashed) Metrics() *Metrics { return hph.metrics }
+
 // SetCollapseTracer sets a callback that will be invoked when a node collapse occurs
 // during commitment calculation. This is used by witness generation to capture paths
 // to HashNodes that need resolution when a FullNode is reduced to a single child.
@@ -1464,6 +1468,7 @@ func (hph *HexPatriciaHashed) unfoldBranchNode(row int, depth int16, deleted boo
 	if err != nil {
 		return err
 	}
+	hph.metrics.AddBranchRead(len(branchData))
 
 	// depthsToTxNum is used for per-file metrics; step is no longer available
 	// from the cache-or-DB helper (cache never had a meaningful step anyway).
@@ -2483,6 +2488,9 @@ func (hph *HexPatriciaHashed) Process(ctx context.Context, updates *Updates, log
 
 	hph.metrics.Reset()
 	hph.metrics.updates.Store(updatesCount)
+	hph.metrics.AddRoundKeys(updatesCount)
+	roundStart := time.Now()
+	defer func() { observeRound(hph.metrics, roundStart) }()
 	if hph.metrics.collectCommitmentMetrics {
 		defer func() {
 			hph.metrics.TotalProcessingTimeInc(start)