From a47789e3d1cde506ebdd61580b701bf7e5eadfe5 Mon Sep 17 00:00:00 2001 From: Kai Aldag Date: Mon, 15 Jun 2026 00:13:08 +0100 Subject: [PATCH 01/26] Phase 2: add spec + roadmap design-of-record The phase-2-spec.md build contract and the detailed Phase 2 design section in ROADMAP.md (decisions locked). The ROADMAP status flips to Done at the end of the phase. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/ROADMAP.md | 126 +++++++++++++++++ docs/phase-2-spec.md | 318 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 444 insertions(+) create mode 100644 docs/phase-2-spec.md diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 6a3ee83..4a834bd 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -168,6 +168,132 @@ the Pillar A rewrite. These are the breaking changes that must precede a 1.0. --- +## Phase 2 — freshness core (detailed, decisions locked) + +Builds the freshness/invalidation control plane **and** the optimistic +verify-and-rerun execution loop on top of it. Out of scope: event-derived +*writes* (Phase 3), the WS ingestion loop and reorg handling (Phase 4). + +### Locked decisions + +1. **`Validity` has three variants** (`EventDriven` dropped — folded into + `Pinned`): `Pinned` (caller-owned: immutable or kept fresh via event writes; + the freshness system never touches it), `Volatile` (governed by the active + policy), `ValidThrough(block)` (pinned until block N, then volatile). Default + is `Volatile`, configurable. +2. **Optimistic verify-and-rerun is in scope.** Don't block on a purge: snapshot, + run sims, and concurrently re-fetch the volatile slots they read (scoped by the + `TxConfig.access_list`); on a value mismatch, refresh and re-run only the + affected sims. Correctness is independent of access-list completeness (the + post-sim actual read-set is re-verified before results are trusted). +3. **Adaptive freshness via the (revived) `SlotObservationTracker`.** Per-slot + `last_value`/`observation_count`/`change_count`/`last_checked`/`last_changed` + drive `should_refetch`. Frequently-changing slots are verified often; stable + ones rarely. +4. **Configurable clock, block-based by default.** `SlotObservationTracker` is made + clock-agnostic (takes `now: u64`); a `FreshnessClock` supplies it — + `BlockClock` (default) or `WallClock` (today's behavior). +5. **Account-level purge.** A fully-volatile address drops account + (balance/nonce/code) + storage via a new `purge_account` primitive; an address + with any pinned slot keeps its account and only its volatile slots are purged. + +### Four-layer model + +| Layer | What | Type | +| --- | --- | --- | +| Classification | `Pinned` / `Volatile` / `ValidThrough` per address/slot | `FreshnessRegistry` | +| Observation | per-slot change-frequency stats (clock-agnostic) | `SlotObservationTracker` (revived) | +| Policy | which volatile slots to verify this cycle, and how | `FreshnessPolicy` trait | +| Mechanism | re-fetch+compare, purge, re-run | `EvmCache` + `FreshnessController` | + +```rust +pub enum Validity { Pinned, Volatile, ValidThrough(u64) } // resolution: slot ▸ account ▸ default + +pub trait FreshnessClock { fn now(&self) -> u64; } // BlockClock (default) | WallClock + +pub trait FreshnessPolicy { + fn select(&mut self, candidates: &[(Address, U256)], + obs: &SlotObservationTracker, now: u64) -> Vec<(Address, U256)>; + fn on_new_block(&mut self, block: u64) {} +} +// built-ins: AlwaysVerify, ObservationDriven (wraps should_refetch), NeverVerify. +// tunable heuristics (min-observations, max-reuse, staleness threshold, …) move +// into a `FreshnessParams` config so users can tune the adaptive model. + +pub struct FreshnessController { /* registry, tracker, policy, clock, fetcher */ } +``` + +### Primitives (on `EvmCache`) + +- `verify_slots(&mut self, slots) -> Vec` — re-fetch current values via + the existing batched `StorageBatchFetchFn`, compare to cached values, inject the + changed ones, and `observe` each (updating the tracker). Returns the changed set. +- `purge_account(&mut self, addr)` — remove `addr` from the CacheDB overlay, the + BlockchainDb accounts map, and its storage, so the next access re-fetches a clean + `AccountInfo`. Distinct from storage-only `purge_pool_storage`. + +### Optimistic execution loop with deferred validation (`FreshnessController::run`) + +`run` returns a `SpeculativeSim { optimistic, validation }` **as soon as the +optimistic sims finish** — it does *not* await RPC. The caller computes against +`optimistic()` immediately and `validate().await`s the verdict when ready. + +```rust +pub struct SpeculativeSim { /* optimistic results + JoinHandle */ } +impl SpeculativeSim { + pub fn optimistic(&self) -> &[SimOutcome]; + pub async fn validate(self) -> Validation; +} +pub enum Validation { + Confirmed, + Corrected { results: Vec, changed: Vec }, + Unverified { reason: String }, +} +``` + +Main thread (`run`): drain pending corrections into the cache → `create_snapshot()` +→ run optimistic sims (capturing read-sets) → **spawn** the validator with `Send` +data only (`Arc`, the `Arc` `StorageBatchFetchFn`, requests, read-sets) +→ return `SpeculativeSim`. + +Background validator (spawned task — never touches the `!Send` cache): `verify_slots` +the predicted volatile set; reconcile by verifying any volatile slot in the actual +read-set not yet checked; if nothing changed → `Confirmed`; else build *corrected* +overlays from the snapshot with the fresh values in their dirty layers, re-run only +the affected sims → `Corrected { results, changed }`. RPC failure → `Unverified`. + +Freshness flow-back: the validator can't mutate the live cache, so `changed` is +returned **and** queued; the next `run` drains the queue and applies it before +snapshotting (eventually-fresh, no cross-thread cache mutation). Dropping a +`SpeculativeSim` aborts the background task. + +Correctness rests on the reconcile step (verify the actual read-set); the access +list only buys the overlap. This `FreshnessController` is the seed of the eventual +`SimulationEngine`. + +### Placement + +`src/cache/freshness.rs` (child of `cache` → reads private layers for enumeration); +`slot_observations.rs` revived + made clock-agnostic; `verify_slots`/`purge_account` +on `EvmCache`. The whole freshness surface lives under the always-on (non-`protocols`) +core. + +### Tests (offline) + +Classification resolution (slot ▸ account ▸ default); observation tracker with an +injected clock (block-based); each built-in policy's `select`; `verify_slots` +against a **stubbed** `StorageBatchFetchFn` returning chosen "current" values +(changed vs unchanged); the full loop — match path (no re-run) and mismatch path +(refresh + selective re-run of only affected sims); `purge_account` drops account + +storage on both layers; `ValidThrough` boundary; `WallClock` vs `BlockClock`. + +### Acceptance + +`cargo fmt --check`, `clippy --all-targets -- -D warnings` (default + +`--lib --no-default-features`), `cargo test`, `RUSTDOCFLAGS=-D warnings cargo doc`. + +--- + ## Key abstractions for later phases (sketches) ```rust diff --git a/docs/phase-2-spec.md b/docs/phase-2-spec.md new file mode 100644 index 0000000..8d6fb3d --- /dev/null +++ b/docs/phase-2-spec.md @@ -0,0 +1,318 @@ +# Phase 2 implementation spec — freshness core + optimistic execution + +Implementation contract for the freshness control plane and the optimistic +verify-and-rerun loop with deferred validation. Read this **with** +[`ROADMAP.md`](ROADMAP.md) (the "Phase 2 — freshness core" section is the design +of record). This document is the precise build contract; where they overlap, +prefer this. + +## 0. Ground rules (non-negotiable) + +- **Branch:** create `phase-2-freshness` off the current `phase-1-engine-seam` + HEAD. Commit there in logical steps. Do **not** push, do **not** tag. Commits + must be unsigned: `git -c commit.gpgsign=false commit …` (the 1Password signing + agent is unavailable here). End every commit message with exactly: + `Co-Authored-By: Claude Opus 4.8 (1M context) ` +- **The whole freshness surface is generic core** — it must compile and lint with + `--no-default-features` (it must NOT depend on the `protocols` feature). +- **Green bar at every commit, both feature configs:** + - `cargo fmt --all --check` + - `cargo clippy --all-targets --no-deps -- -D warnings` + - `cargo clippy --lib --no-default-features --no-deps -- -D warnings` + - `cargo test` + - `RUSTDOCFLAGS="-D warnings" cargo doc --no-deps` +- MSRV is 1.88 — no newer-than-1.88 std APIs. Edition 2024. +- Do **not** change Phase 1 behavior or break any existing test (118 + doctests). +- No new dependencies without strong justification (tokio is already present with + `rt-multi-thread` + `macros` in dev). The async loop uses tokio (already a dep). + +## 1. Objective & scope + +Deliver the four-layer freshness model and the optimistic execution loop: + +- **Classification** — `Validity` (`Pinned`/`Volatile`/`ValidThrough`) + `FreshnessRegistry`. +- **Observation** — revive `SlotObservationTracker`, make it clock-agnostic. +- **Policy** — `FreshnessPolicy` trait + `AlwaysVerify`/`ObservationDriven`/`NeverVerify`. +- **Mechanism** — `EvmCache::verify_slots` + `purge_account`; `FreshnessController` + running the optimistic loop returning `SpeculativeSim` (deferred validation). + +**In scope:** optimistic verification of the **storage-slot** read-set, deferred +validation (`SpeculativeSim`/`Validation`), background re-run on mismatch, +configurable block/wall clock, the `purge_account` primitive, overlay read-set +capture. + +**Out of scope (document as follow-ups, do not build):** account-*balance* +optimistic verification (needs a batched balance fetcher — the current +`StorageBatchFetchFn` is storage-only); event-derived writes (Phase 3); WS +ingestion / reorgs / RPC reconciliation (Phase 4). Committing simulations +speculatively is out of scope — the optimistic loop handles **non-committing** +evaluation sims only. + +## 2. Reuse these existing pieces (do not reinvent) + +- `cache::EvmCache` (`src/cache/mod.rs`): `create_snapshot() -> Arc`, + `storage_batch_fetcher() -> Option<&StorageBatchFetchFn>`, + `inject_storage_batch(&[(Address,U256,U256)])`, `purge_pool_storage`, + `purge_pool_slots`, `call_raw_with`/`TxConfig`, `CallSimulationResult`, + `blockchain_db()`, `db_mut()`. +- `cache::EvmOverlay` / `cache::EvmSnapshot` (`overlay.rs`/`snapshot.rs`): + `EvmOverlay::new(Arc, Option)`, `call_raw`, + `simulate_with_transfer_tracking`. `EvmOverlay` is `Send`. +- `cache::SlotObservationTracker` (`src/cache/slot_observations.rs`): **dormant** — + this is its intended use. `SlotObservation { last_value, observation_count, + change_count, last_checked, last_changed }`, `observe`, `should_refetch`, + `take_skipped`, persistence. +- `StorageBatchFetchFn = Arc) -> Vec<(Address,U256,Result)> + Send + Sync>` + — the batched RPC fetcher. **Synchronous** (it block_on's internally), `Send + Sync`. +- `access_set::StorageAccessList { accounts: HashSet
, slots: HashSet<(Address,U256)> }`. + +## 3. Module layout + +- **`src/freshness.rs`** (new, top-level, generic): `Validity`, `FreshnessRegistry`, + `FreshnessClock` + `BlockClock` + `WallClock`, `FreshnessParams`, + `FreshnessPolicy` + built-ins, `SlotChange`, `Validation`, `SpeculativeSim`, + `SimRequest`, `FreshnessController`. Operates on `EvmCache` via its public API. +- **`src/cache/mod.rs`**: add `verify_slots`, `purge_account`, + `set_storage_batch_fetcher` (test seam). +- **`src/cache/overlay.rs`**: add `call_raw_with_access_list` (read-set capture). +- **`src/cache/slot_observations.rs`**: make clock-agnostic (take `now: u64`). +- **`src/lib.rs`**: `pub mod freshness;` + re-export the key types. + +## 4. Types & behavior + +### 4.1 Classification + +```rust +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Validity { Pinned, Volatile, ValidThrough(u64) } + +#[derive(Clone, Debug)] +pub struct FreshnessRegistry { + default: Validity, // Volatile by default + accounts: HashMap, + slots: HashMap<(Address, U256), Validity>, +} +``` +- `new()` → default `Volatile`; `with_default(Validity)`. +- Builder-style setters returning `&mut Self`: `pin`, `pin_slot`, `mark_volatile`, + `mark_volatile_slot`, `valid_through`, `valid_through_slot`, `set_account`, `set_slot`. +- `validity(addr, slot) -> Validity` — resolution **slot ▸ account ▸ default**. +- `is_volatile(addr, slot, now: u64) -> bool` — `true` for `Volatile`, and for + `ValidThrough(m)` when `now > m`; `false` for `Pinned` / still-valid `ValidThrough`. +- Must be `Clone` (background task needs a snapshot of it). + +### 4.2 Clock + +```rust +pub trait FreshnessClock: Send + Sync { fn now(&self) -> u64; } +pub struct BlockClock(Arc); // settable via set_block(u64); Clone shares the Arc +pub struct WallClock; // now() = unix seconds +``` +`BlockClock` is the default. The controller calls `clock.now()` and threads it as +`now: u64` everywhere (tracker, policy, `is_volatile`). + +### 4.3 Observation tracker (revive + clock-agnostic) + +Change `SlotObservationTracker` so it does **not** call `unix_now()` internally: +- `observe(&mut self, addr, slot, value, now: u64) -> bool` +- `should_refetch(&self, addr, slot, now: u64, params: &FreshnessParams) -> bool` + +Move the hardcoded thresholds into `FreshnessParams`: +```rust +#[derive(Clone, Debug)] +pub struct FreshnessParams { + pub min_observations: u32, // default 10 + pub max_reuse: u64, // clock units; block default e.g. 300; wall = 7*86400 + pub staleness_threshold: f64, // default 0.05 + pub always_refetch_rate: f64, // default 0.9 + pub cycle_interval: u64, // clock units per "cycle"; block default 1 +} +``` +`should_refetch` keeps the existing probabilistic logic but in clock units. Update +the existing `slot_observations.rs` unit tests to pass `now`/`params`. + +### 4.4 Policy + +```rust +pub trait FreshnessPolicy: Send { + /// Of these volatile candidate slots, which must be verified this cycle? + fn select(&mut self, candidates: &[(Address, U256)], + obs: &SlotObservationTracker, now: u64) -> Vec<(Address, U256)>; + fn on_new_block(&mut self, _block: u64) {} +} +``` +Built-ins: +- `AlwaysVerify` — returns all candidates (safe/eager). +- `NeverVerify` — returns empty (trust-all; results always `Confirmed`). +- `ObservationDriven { params: FreshnessParams }` — returns candidates where + `obs.should_refetch(addr, slot, now, ¶ms)`. + +### 4.5 Results & deferred validation + +```rust +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SlotChange { pub address: Address, pub slot: U256, pub old: U256, pub new: U256 } + +pub enum Validation { + Confirmed, + Corrected { results: Vec, changed: Vec }, + Unverified { reason: String }, +} + +pub struct SpeculativeSim { + optimistic: Vec, + validation: tokio::task::JoinHandle, +} +impl SpeculativeSim { + pub fn optimistic(&self) -> &[CallSimulationResult]; + pub fn into_optimistic(self) -> Vec; // aborts validation + pub async fn validate(self) -> Validation; // awaits the verdict +} +impl Drop for SpeculativeSim { /* abort the background task */ } +``` +`CallSimulationResult` must be `Clone` (verify it already is; add derive if needed) +so optimistic + corrected copies can coexist and cross the task boundary. + +### 4.6 Request + +```rust +pub struct SimRequest { + pub from: Address, + pub to: Address, + pub calldata: Bytes, + pub tx: TxConfig, // access_list here is the predicted read set (perf hint) +} +``` + +## 5. `EvmCache` primitives + +- `verify_slots(&mut self, slots: &[(Address, U256)]) -> anyhow::Result>`: + fetch fresh values via the batch fetcher; compare to currently-cached values; for + each that differs, `inject_storage_batch` the fresh value and record a `SlotChange`. + Returns the changed set. (Synchronous main-thread helper + the test target.) +- `purge_account(&mut self, addr: Address)`: remove `addr` from the CacheDB overlay + accounts (`self.db.cache.accounts`), the BlockchainDb accounts map, and the + BlockchainDb storage map — so the next access re-fetches a clean `AccountInfo`. + Distinct from storage-only `purge_pool_storage`. Add a doc comment + a test. +- `set_storage_batch_fetcher(&mut self, f: StorageBatchFetchFn)`: test/extensibility + seam so a stub fetcher can be injected without a provider. + +## 6. `EvmOverlay` read-set capture + +Add `call_raw_with_access_list(&mut self, from, to, calldata) -> Result<(ExecutionResult, StorageAccessList)>` +mirroring `EvmCache::call_raw_with_access_list`: run non-committing, extract touched +accounts/slots from the journaled state before reverting. This is the per-sim read +set the reconcile step needs. + +## 7. `FreshnessController` + the optimistic loop + +```rust +pub struct FreshnessController { + registry: FreshnessRegistry, + tracker: Arc>, + policy: P, + clock: C, + params: FreshnessParams, + pending: Arc>>, // corrections flowing back from bg tasks +} +``` + +`run(&mut self, cache: &mut EvmCache, requests: Vec) -> Result` +(main thread): +1. **Drain `pending`** into `cache.inject_storage_batch(...)` (apply corrections from + prior cycles before snapshotting). +2. `let snapshot = cache.create_snapshot();` and grab + `let fetcher = cache.storage_batch_fetcher().cloned();` (the Arc fetcher). +3. **Optimistic sims:** for each request, build an `EvmOverlay::new(snapshot.clone(), None)` + and run `call_raw_with_access_list` → collect `optimistic: Vec` + and per-sim actual volatile read-sets (touched slots filtered by + `registry.is_volatile(addr, slot, now)`). +4. **Predicted candidates:** union of each request's `tx.access_list` slots filtered + to volatile; `policy.select(candidates, &tracker.lock(), now)` → the verify set. +5. **Spawn the validator** (`tokio::spawn`) with `Send` data only: `snapshot` (Arc), + `fetcher` (Arc), the requests, the per-sim read-sets, a `registry.clone()`, the + `tracker` (Arc), the `pending` (Arc), `now`. Return `SpeculativeSim` + immediately. + +**Background validator** (must touch **no** `!Send` state — only the Arc/Send data): +1. `verify` = the policy-selected set ∪ (each sim's actual volatile read-set). Call + the `fetcher` for those slots; compare each to the snapshot's value + (`snapshot` exposes its slot values — add a crate-internal accessor if needed). +2. `observe` every checked slot into the `tracker` (lock); collect `changed: Vec`. +3. If `changed` empty → `Validation::Confirmed`. +4. Else: push `changed` into `pending` (flow-back); build corrected overlays + (`EvmOverlay::new(snapshot.clone(), None)` then write the fresh values into the + overlay via a dirty-layer override — add an `EvmOverlay::override_slot(addr,slot,value)` + if needed); re-run **only** the requests whose read-set intersects `changed`; + return `Validation::Corrected { results, changed }` (results = optimistic with the + re-run ones replaced). +5. On fetcher error → `Validation::Unverified { reason }` (do not trust silently). + +`on_new_block(&mut self, block: u64)`: `clock` advance (if `BlockClock`), `policy.on_new_block(block)`. + +**Concurrency notes:** `tracker` and `pending` are `Arc>` so the background +task updates them safely; the live `EvmCache` is never shared across threads. +`run` requires a multi-thread tokio runtime (document it; mirror the Phase-1 +constructor note). The `fetcher` is synchronous (block_in_place internally) and is +fine to call from the spawned task. + +## 8. Tests (offline, no network) + +All via a **stubbed** `StorageBatchFetchFn` (`set_storage_batch_fetcher`) returning +chosen "current" values; build the cache over the mocked provider (see +`tests/common`/`examples/support` patterns). Cover: + +- `FreshnessRegistry`: resolution order (slot ▸ account ▸ default); `is_volatile` + for each variant incl. `ValidThrough` boundary at `now == m` vs `now > m`; + `with_default` non-default. +- `SlotObservationTracker` (clock-agnostic): `observe` change detection with explicit + `now`; `should_refetch` for unknown / insufficient / never-changed / always-changed + with a `FreshnessParams`; existing tests updated to the new signatures. +- Each policy's `select`: `AlwaysVerify` (all), `NeverVerify` (none), + `ObservationDriven` (only `should_refetch` slots). +- `EvmCache::verify_slots` against a stub fetcher: changed vs unchanged; assert it + injects fresh values and returns the right `SlotChange`s. +- `EvmCache::purge_account`: account + storage gone from both layers. +- `EvmOverlay::call_raw_with_access_list`: returns the touched slots/accounts. +- **The full loop** (`FreshnessController::run` on a multi-thread test runtime, + stub fetcher): (a) **match path** — fetcher returns unchanged values → + `Validation::Confirmed`, optimistic == nothing re-run; (b) **mismatch path** — + fetcher returns a changed value for a slot a sim read → `Validation::Corrected` + with corrected results differing from optimistic, and only the affected sim re-run; + (c) `optimistic()` is readable before `validate()`; (d) `pending` drained on the + next `run`; (e) `Unverified` when the stub returns an error. +- `BlockClock` vs `WallClock` selection behavior. + +Put unit tests in-module (`#[cfg(test)]`) and the loop/integration tests in +`tests/freshness.rs` (shared `tests/common` helpers; add a stub-fetcher helper). + +## 9. Docs & example + +- Rustdoc on every public item (CI runs `-D warnings`; there is no `missing_docs` + gate, but document thoroughly anyway). +- A module-level `//!` doc on `freshness.rs` explaining the four layers + the + optimistic/deferred-validation model, with a short runnable doctest for the + registry + policy (no network). +- An offline example `examples/freshness_optimistic.rs` (using `examples/support`) + that: builds a cache, registers a pinned + a volatile slot, runs a `SimRequest` + through a `FreshnessController` with a **stub fetcher** that reports one slot + changed, and prints the `optimistic()` result then the `Validation` (showing a + `Corrected`). Add it to the README example table. + +## 10. Build order (commit per step, green each time) + +1. Clock-agnostic `SlotObservationTracker` + `FreshnessParams` (update its tests). +2. `Validity` + `FreshnessRegistry` + `FreshnessClock`/`BlockClock`/`WallClock` + + `FreshnessPolicy` + built-ins (with unit tests). +3. `EvmCache::verify_slots` + `purge_account` + `set_storage_batch_fetcher`; + `EvmOverlay::call_raw_with_access_list` (with tests). +4. `FreshnessController` + `SpeculativeSim`/`Validation` optimistic loop (with the + full-loop tests). +5. Docs + example + README + lib re-exports; update `docs/ROADMAP.md` Phase 2 status + to "Done". + +## 11. Final acceptance + +Both feature configs green (§0). All new + existing tests pass. The example runs +offline and demonstrates a `Corrected` validation. Report: what landed per file, +the public API added, test coverage, and the verification output. From dc4733640fbbb07dfbd901ff443c06867afc0bb4 Mon Sep 17 00:00:00 2001 From: Kai Aldag Date: Mon, 15 Jun 2026 00:17:04 +0100 Subject: [PATCH 02/26] Phase 2 (step 1): clock-agnostic SlotObservationTracker + FreshnessParams Make the observation tracker take an explicit `now: u64` (clock units) on `observe` and `should_refetch`, and move the hardcoded thresholds into a new `freshness::FreshnessParams` (block-oriented defaults, plus `for_wall_clock` helper). Drops the internal `unix_now` so the tracker is driven by a configurable clock. Updates the in-module tests to the new signatures and adds clock-recording / max-reuse coverage. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/cache/slot_observations.rs | 167 ++++++++++++++++++++------------- src/freshness.rs | 75 +++++++++++++++ src/lib.rs | 1 + 3 files changed, 177 insertions(+), 66 deletions(-) create mode 100644 src/freshness.rs diff --git a/src/cache/slot_observations.rs b/src/cache/slot_observations.rs index 8f12778..7f54a2d 100644 --- a/src/cache/slot_observations.rs +++ b/src/cache/slot_observations.rs @@ -5,42 +5,35 @@ //! time. Slots that change frequently are rechecked sooner; stable slots are //! trusted longer (subject to a maximum age). The observations are persisted to //! disk so the heuristics survive across runs. +//! +//! # Clock-agnostic +//! +//! The tracker does not read the wall clock itself: callers pass `now` (in +//! clock units) into [`observe`](SlotObservationTracker::observe) and +//! [`should_refetch`](SlotObservationTracker::should_refetch), and the thresholds +//! live in a [`crate::freshness::FreshnessParams`]. This lets the freshness +//! controller drive the tracker from either a block clock or a wall clock. -use std::{ - collections::HashMap, - path::Path, - time::{SystemTime, UNIX_EPOCH}, -}; +use std::{collections::HashMap, path::Path}; use alloy_primitives::{Address, U256}; use serde::{Deserialize, Serialize}; use tracing::{debug, warn}; -/// Minimum observations before we trust the change frequency data. -const MIN_OBSERVATIONS: u32 = 10; - -/// Maximum time (seconds) to reuse a cached slot value before rechecking. -/// Even never-changed slots get rechecked after 1 week. -const MAX_REUSE_SECS: u64 = 7 * 86400; - -/// Refetch threshold: if expected probability of change exceeds this, refetch. -const STALENESS_THRESHOLD: f64 = 0.05; - -/// Slots that change more than 90% of the time are always refetched. -const ALWAYS_REFETCH_RATE: f64 = 0.9; - -/// Estimated cycle interval in seconds (used for probabilistic model). -const ESTIMATED_CYCLE_SECS: f64 = 60.0; +use crate::freshness::FreshnessParams; /// Per-slot observation record, persisted to disk. #[derive(Serialize, Deserialize, Clone, Debug)] pub struct SlotObservation { + /// Most recently observed slot value. pub last_value: U256, + /// Total number of times this slot has been observed. pub observation_count: u32, + /// Number of observations that differed from the previous value. pub change_count: u32, - /// Unix timestamp of most recent observation. + /// Clock value (block number or unix seconds) of the most recent observation. pub last_checked: u64, - /// Unix timestamp of most recent value change. + /// Clock value of the most recent value change. pub last_changed: u64, } @@ -120,7 +113,17 @@ impl SlotObservationTracker { /// /// Returns `true` if the slot should be purged and re-fetched. /// Returns `false` if the cached value is likely still valid. - pub fn should_refetch(&self, addr: Address, slot: U256) -> bool { + /// + /// `now` is the current clock value (block number or unix seconds) and + /// `params` carries the (clock-unit) thresholds — see + /// [`crate::freshness::FreshnessParams`]. + pub fn should_refetch( + &self, + addr: Address, + slot: U256, + now: u64, + params: &FreshnessParams, + ) -> bool { let key = SlotKey { address: addr, slot, @@ -129,19 +132,17 @@ impl SlotObservationTracker { return true; // never observed → must fetch }; - let now = unix_now(); - // Always refetch if insufficient data to make predictions - if obs.observation_count < MIN_OBSERVATIONS { + if obs.observation_count < params.min_observations { return true; } - // Always refetch if last check was > 1 week ago - if now.saturating_sub(obs.last_checked) > MAX_REUSE_SECS { + // Always refetch if last check was longer than the reuse window ago + if now.saturating_sub(obs.last_checked) > params.max_reuse { return true; } - // Never-changed slots: reuse up to the 1-week max + // Never-changed slots: reuse up to the max-reuse window if obs.change_count == 0 { return false; } @@ -149,26 +150,27 @@ impl SlotObservationTracker { let change_rate = obs.change_count as f64 / obs.observation_count as f64; // Always-changing slots: always refetch - if change_rate > ALWAYS_REFETCH_RATE { + if change_rate > params.always_refetch_rate { return true; } // Probabilistic: estimate expected changes since last check - let secs_elapsed = now.saturating_sub(obs.last_checked) as f64; - let cycles_elapsed = (secs_elapsed / ESTIMATED_CYCLE_SECS).max(1.0); + let units_elapsed = now.saturating_sub(obs.last_checked) as f64; + let cycle_interval = params.cycle_interval.max(1) as f64; + let cycles_elapsed = (units_elapsed / cycle_interval).max(1.0); let expected_changes = change_rate * cycles_elapsed; - expected_changes > STALENESS_THRESHOLD + expected_changes > params.staleness_threshold } /// Record a fresh observation after re-fetch or injection. /// + /// `now` is the current clock value (block number or unix seconds). /// Returns `true` if the value changed from the last observation. - pub fn observe(&mut self, addr: Address, slot: U256, value: U256) -> bool { + pub fn observe(&mut self, addr: Address, slot: U256, value: U256, now: u64) -> bool { let key = SlotKey { address: addr, slot, }; - let now = unix_now(); self.dirty = true; match self.observations.get_mut(&key) { @@ -249,13 +251,6 @@ impl Default for SlotObservationTracker { } } -fn unix_now() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_secs() -} - #[cfg(test)] mod tests { use super::*; @@ -264,47 +259,71 @@ mod tests { Address::new([n; 20]) } + /// Block-clock params with a 1-unit cycle so each `observe` advances exactly + /// one cycle — keeps the probabilistic arithmetic easy to reason about. + fn params() -> FreshnessParams { + FreshnessParams::default() + } + #[test] fn test_unknown_slot_always_refetches() { let tracker = SlotObservationTracker::new(); - assert!(tracker.should_refetch(addr(1), U256::from(0))); + assert!(tracker.should_refetch(addr(1), U256::from(0), 100, ¶ms())); } #[test] fn test_insufficient_observations_refetches() { let mut tracker = SlotObservationTracker::new(); + let p = params(); let a = addr(1); let slot = U256::from(4); - // Record fewer than MIN_OBSERVATIONS observations - for _ in 0..(MIN_OBSERVATIONS - 1) { - tracker.observe(a, slot, U256::from(42)); + // Record fewer than `min_observations` observations. + for now in 0..(p.min_observations - 1) { + tracker.observe(a, slot, U256::from(42), now as u64); } - assert!(tracker.should_refetch(a, slot)); + assert!(tracker.should_refetch(a, slot, p.min_observations as u64, &p)); } #[test] fn test_never_changed_slot_skips_refetch() { let mut tracker = SlotObservationTracker::new(); + let p = params(); let a = addr(1); let slot = U256::from(4); let value = U256::from(42); - // Build up enough observations with the same value - for _ in 0..MIN_OBSERVATIONS { - tracker.observe(a, slot, value); + // Build up enough observations with the same value at consecutive ticks. + for now in 0..p.min_observations { + tracker.observe(a, slot, value, now as u64); } - assert!(!tracker.should_refetch(a, slot)); + // Re-check immediately after the last observation (within the reuse window). + assert!(!tracker.should_refetch(a, slot, p.min_observations as u64 - 1, &p)); + } + + #[test] + fn test_never_changed_slot_refetches_past_max_reuse() { + let mut tracker = SlotObservationTracker::new(); + let p = params(); + let a = addr(1); + let slot = U256::from(4); + for now in 0..p.min_observations { + tracker.observe(a, slot, U256::from(42), now as u64); + } + // Far past the reuse window even a never-changed slot is rechecked. + let now = p.min_observations as u64 + p.max_reuse + 1; + assert!(tracker.should_refetch(a, slot, now, &p)); } #[test] fn test_always_changing_slot_refetches() { let mut tracker = SlotObservationTracker::new(); + let p = params(); let a = addr(1); let slot = U256::from(4); - // Record MIN_OBSERVATIONS observations, each with a different value - for i in 0..(MIN_OBSERVATIONS + 1) { - tracker.observe(a, slot, U256::from(i)); + // Record observations, each with a different value, at consecutive ticks. + for now in 0..(p.min_observations + 1) { + tracker.observe(a, slot, U256::from(now), now as u64); } - assert!(tracker.should_refetch(a, slot)); + assert!(tracker.should_refetch(a, slot, p.min_observations as u64 + 1, &p)); } #[test] @@ -312,24 +331,40 @@ mod tests { let mut tracker = SlotObservationTracker::new(); let a = addr(1); let slot = U256::from(0); - assert!(!tracker.observe(a, slot, U256::from(1))); // first = baseline - assert!(!tracker.observe(a, slot, U256::from(1))); // same - assert!(tracker.observe(a, slot, U256::from(2))); // changed - assert!(!tracker.observe(a, slot, U256::from(2))); // same again + assert!(!tracker.observe(a, slot, U256::from(1), 0)); // first = baseline + assert!(!tracker.observe(a, slot, U256::from(1), 1)); // same + assert!(tracker.observe(a, slot, U256::from(2), 2)); // changed + assert!(!tracker.observe(a, slot, U256::from(2), 3)); // same again + } + + #[test] + fn test_observe_records_change_clock() { + let mut tracker = SlotObservationTracker::new(); + let a = addr(1); + let slot = U256::from(0); + tracker.observe(a, slot, U256::from(1), 10); // baseline at tick 10 + tracker.observe(a, slot, U256::from(2), 25); // change at tick 25 + let key = SlotKey { address: a, slot }; + let obs = &tracker.observations[&key]; + assert_eq!(obs.last_checked, 25); + assert_eq!(obs.last_changed, 25); + assert_eq!(obs.change_count, 1); + assert_eq!(obs.observation_count, 2); } #[test] fn test_reset_contract_clears_observations() { let mut tracker = SlotObservationTracker::new(); + let p = params(); let a = addr(1); - for i in 0..MIN_OBSERVATIONS { - tracker.observe(a, U256::from(i), U256::from(42)); + for i in 0..p.min_observations { + tracker.observe(a, U256::from(i), U256::from(42), i as u64); } assert!(!tracker.is_empty()); tracker.reset_contract(a); assert_eq!(tracker.len(), 0); // After reset, should_refetch returns true - assert!(tracker.should_refetch(a, U256::from(0))); + assert!(tracker.should_refetch(a, U256::from(0), 100, &p)); } #[test] @@ -362,8 +397,8 @@ mod tests { let mut tracker = SlotObservationTracker::new(); let a = addr(1); - tracker.observe(a, U256::from(0), U256::from(100)); - tracker.observe(a, U256::from(4), U256::from(200)); + tracker.observe(a, U256::from(0), U256::from(100), 0); + tracker.observe(a, U256::from(4), U256::from(200), 0); tracker.save(&path).unwrap(); let loaded = SlotObservationTracker::load(&path); @@ -380,9 +415,9 @@ mod tests { let mut tracker = SlotObservationTracker::new(); let a = addr(1); assert_eq!(tracker.last_value(a, U256::from(0)), None); - tracker.observe(a, U256::from(0), U256::from(42)); + tracker.observe(a, U256::from(0), U256::from(42), 0); assert_eq!(tracker.last_value(a, U256::from(0)), Some(U256::from(42))); - tracker.observe(a, U256::from(0), U256::from(99)); + tracker.observe(a, U256::from(0), U256::from(99), 1); assert_eq!(tracker.last_value(a, U256::from(0)), Some(U256::from(99))); } } diff --git a/src/freshness.rs b/src/freshness.rs new file mode 100644 index 0000000..c264e03 --- /dev/null +++ b/src/freshness.rs @@ -0,0 +1,75 @@ +//! Freshness control plane and the optimistic verify-and-rerun execution loop. +//! +//! This module is the generic core of the engine's "honest freshness" model. +//! The four-layer model, the policy traits, and the optimistic freshness +//! controller are built up across the Phase 2 steps; this step introduces the +//! clock-agnostic [`FreshnessParams`] that tune the adaptive +//! [`SlotObservationTracker`](crate::cache::SlotObservationTracker). + +/// Default minimum observations before the change-frequency data is trusted. +pub const DEFAULT_MIN_OBSERVATIONS: u32 = 10; + +/// Default maximum reuse window, in clock units, before a slot is rechecked. +/// +/// Block-based default (≈300 blocks). Wall-clock users typically set this to +/// `7 * 86400` (one week) to reproduce the original behavior. +pub const DEFAULT_MAX_REUSE: u64 = 300; + +/// Default refetch threshold on expected probability of change. +pub const DEFAULT_STALENESS_THRESHOLD: f64 = 0.05; + +/// Default change-rate above which a slot is always refetched. +pub const DEFAULT_ALWAYS_REFETCH_RATE: f64 = 0.9; + +/// Default clock units per "cycle" used by the probabilistic model. +pub const DEFAULT_CYCLE_INTERVAL: u64 = 1; + +/// Tunable thresholds for the adaptive freshness model. +/// +/// All time-like fields are expressed in **clock units** (`FreshnessClock`): +/// block numbers for a block clock, unix seconds for a wall clock. The defaults +/// are block-oriented; wall-clock users should raise [`max_reuse`](Self::max_reuse) +/// and [`cycle_interval`](Self::cycle_interval) accordingly. +#[derive(Clone, Debug, PartialEq)] +pub struct FreshnessParams { + /// Minimum observations before the change frequency is trusted (else refetch). + pub min_observations: u32, + /// Maximum reuse window (clock units) before a slot is force-rechecked. + pub max_reuse: u64, + /// Refetch when the expected probability of change exceeds this threshold. + pub staleness_threshold: f64, + /// Slots changing more often than this rate are always refetched. + pub always_refetch_rate: f64, + /// Clock units per "cycle" for the probabilistic expected-change estimate. + /// Must be non-zero; a zero is treated as one to avoid division by zero. + pub cycle_interval: u64, +} + +impl Default for FreshnessParams { + fn default() -> Self { + Self { + min_observations: DEFAULT_MIN_OBSERVATIONS, + max_reuse: DEFAULT_MAX_REUSE, + staleness_threshold: DEFAULT_STALENESS_THRESHOLD, + always_refetch_rate: DEFAULT_ALWAYS_REFETCH_RATE, + cycle_interval: DEFAULT_CYCLE_INTERVAL, + } + } +} + +impl FreshnessParams { + /// Block-oriented defaults (`max_reuse ≈ 300` blocks, one cycle per block). + pub fn for_block_clock() -> Self { + Self::default() + } + + /// Wall-clock defaults: reuse up to one week, ~60s cycles, matching the + /// original (pre-Phase-2) hardcoded behavior of the observation tracker. + pub fn for_wall_clock() -> Self { + Self { + max_reuse: 7 * 86400, + cycle_interval: 60, + ..Self::default() + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 51104e1..15711cb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -34,6 +34,7 @@ pub mod cache; pub mod create3; pub mod deploy; pub mod errors; +pub mod freshness; pub mod inspector; pub mod multicall; pub mod prefetch_registry; From bffdb15f34c9c6d6ee2dc1e292bddc57f4a777e4 Mon Sep 17 00:00:00 2001 From: Kai Aldag Date: Mon, 15 Jun 2026 00:19:48 +0100 Subject: [PATCH 03/26] Phase 2 (step 2): Validity/FreshnessRegistry, clocks, policies Add the classification layer (`Validity` + `FreshnessRegistry` with slot-account-default resolution and `is_volatile`), the configurable clock (`FreshnessClock` + `BlockClock`/`WallClock`), and the `FreshnessPolicy` trait with `AlwaysVerify`/`NeverVerify`/`ObservationDriven` built-ins. All generic core (no `protocols` dep). Unit tests cover resolution order, the ValidThrough boundary, clock sharing, and each policy's select. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/freshness.rs | 477 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 472 insertions(+), 5 deletions(-) diff --git a/src/freshness.rs b/src/freshness.rs index c264e03..d784a70 100644 --- a/src/freshness.rs +++ b/src/freshness.rs @@ -1,10 +1,64 @@ //! Freshness control plane and the optimistic verify-and-rerun execution loop. //! -//! This module is the generic core of the engine's "honest freshness" model. -//! The four-layer model, the policy traits, and the optimistic freshness -//! controller are built up across the Phase 2 steps; this step introduces the -//! clock-agnostic [`FreshnessParams`] that tune the adaptive -//! [`SlotObservationTracker`](crate::cache::SlotObservationTracker). +//! This module is the generic core of the engine's "honest freshness" model: it +//! knows which cached state it can trust, for how long, and how to keep the rest +//! correct without blocking simulations on RPC. It is built from four layers: +//! +//! 1. **Classification** — [`Validity`] (`Pinned` / `Volatile` / `ValidThrough`) +//! and the [`FreshnessRegistry`] that resolves a validity per `(address, slot)` +//! with the precedence **slot ▸ account ▸ default**. +//! 2. **Observation** — [`SlotObservationTracker`] records per-slot change +//! frequency (clock-agnostic) to drive adaptive re-verification, tuned by +//! [`FreshnessParams`]. +//! 3. **Policy** — the [`FreshnessPolicy`] trait decides *which* volatile slots to +//! verify this cycle; built-ins are [`AlwaysVerify`], [`NeverVerify`] and +//! [`ObservationDriven`]. +//! 4. **Mechanism** — `EvmCache::verify_slots` / `EvmCache::purge_account`, and +//! the freshness controller that runs the optimistic loop. +//! +//! The clock is configurable via [`FreshnessClock`]: [`BlockClock`] (the default, +//! block-number based) or [`WallClock`] (unix seconds). The controller threads +//! `clock.now()` as `now: u64` through the tracker, the policy, and +//! [`FreshnessRegistry::is_volatile`]. +//! +//! # Example +//! +//! Classification + policy selection, no network required: +//! +//! ``` +//! use alloy_primitives::{Address, U256}; +//! use evm_fork_cache::freshness::{ +//! AlwaysVerify, FreshnessPolicy, FreshnessRegistry, NeverVerify, +//! }; +//! use evm_fork_cache::cache::SlotObservationTracker; +//! +//! let pool = Address::repeat_byte(0x01); +//! let slot0 = U256::from(0); +//! let immutable = U256::from(6); // e.g. token0 +//! +//! let mut registry = FreshnessRegistry::new(); // default: Volatile +//! registry.pin_slot(pool, immutable); // never re-verified +//! +//! // `now` is in clock units (block number for the default BlockClock). +//! let now = 100; +//! assert!(registry.is_volatile(pool, slot0, now)); +//! assert!(!registry.is_volatile(pool, immutable, now)); +//! +//! // Policies pick which volatile candidates to verify this cycle. +//! let obs = SlotObservationTracker::new(); +//! let candidates = [(pool, slot0)]; +//! assert_eq!(AlwaysVerify.select(&candidates, &obs, now), vec![(pool, slot0)]); +//! assert!(NeverVerify.select(&candidates, &obs, now).is_empty()); +//! ``` + +use std::collections::HashMap; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use alloy_primitives::{Address, U256}; + +use crate::cache::SlotObservationTracker; /// Default minimum observations before the change-frequency data is trusted. pub const DEFAULT_MIN_OBSERVATIONS: u32 = 10; @@ -73,3 +127,416 @@ impl FreshnessParams { } } } + +// --------------------------------------------------------------------------- +// 1. Classification +// --------------------------------------------------------------------------- + +/// How long a cached account or storage slot can be trusted. +/// +/// Resolution precedence is **slot ▸ account ▸ default** (see +/// [`FreshnessRegistry::validity`]). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Validity { + /// Caller-owned: immutable, or kept fresh out-of-band (e.g. via event + /// writes). The freshness system never re-verifies or purges it. + Pinned, + /// Governed by the active [`FreshnessPolicy`]; may be re-verified each cycle. + Volatile, + /// Pinned until clock value `N` (inclusive), then treated as [`Volatile`]. + /// + /// [`Volatile`]: Validity::Volatile + ValidThrough(u64), +} + +/// Per-address / per-slot validity classification. +/// +/// A slot's validity is resolved with the precedence **slot ▸ account ▸ +/// default**: an explicit `(address, slot)` entry wins, else the account-level +/// entry for `address`, else the registry default ([`Validity::Volatile`] unless +/// changed via [`with_default`](Self::with_default)). +/// +/// The setters are builder-style (`&mut Self`) so they can be chained. +#[derive(Clone, Debug)] +pub struct FreshnessRegistry { + default: Validity, + accounts: HashMap, + slots: HashMap<(Address, U256), Validity>, +} + +impl Default for FreshnessRegistry { + fn default() -> Self { + Self::new() + } +} + +impl FreshnessRegistry { + /// A registry whose default validity is [`Validity::Volatile`]. + pub fn new() -> Self { + Self { + default: Validity::Volatile, + accounts: HashMap::new(), + slots: HashMap::new(), + } + } + + /// A registry with a custom default validity for unclassified state. + pub fn with_default(default: Validity) -> Self { + Self { + default, + accounts: HashMap::new(), + slots: HashMap::new(), + } + } + + /// The default validity applied when neither the slot nor the account is set. + pub fn default_validity(&self) -> Validity { + self.default + } + + /// Pin an account ([`Validity::Pinned`]). + pub fn pin(&mut self, addr: Address) -> &mut Self { + self.set_account(addr, Validity::Pinned) + } + + /// Pin a single slot ([`Validity::Pinned`]). + pub fn pin_slot(&mut self, addr: Address, slot: U256) -> &mut Self { + self.set_slot(addr, slot, Validity::Pinned) + } + + /// Mark an account [`Validity::Volatile`]. + pub fn mark_volatile(&mut self, addr: Address) -> &mut Self { + self.set_account(addr, Validity::Volatile) + } + + /// Mark a single slot [`Validity::Volatile`]. + pub fn mark_volatile_slot(&mut self, addr: Address, slot: U256) -> &mut Self { + self.set_slot(addr, slot, Validity::Volatile) + } + + /// Mark an account [`Validity::ValidThrough`] block/clock `n`. + pub fn valid_through(&mut self, addr: Address, n: u64) -> &mut Self { + self.set_account(addr, Validity::ValidThrough(n)) + } + + /// Mark a single slot [`Validity::ValidThrough`] block/clock `n`. + pub fn valid_through_slot(&mut self, addr: Address, slot: U256, n: u64) -> &mut Self { + self.set_slot(addr, slot, Validity::ValidThrough(n)) + } + + /// Set the account-level validity for `addr`. + pub fn set_account(&mut self, addr: Address, validity: Validity) -> &mut Self { + self.accounts.insert(addr, validity); + self + } + + /// Set the slot-level validity for `(addr, slot)`. + pub fn set_slot(&mut self, addr: Address, slot: U256, validity: Validity) -> &mut Self { + self.slots.insert((addr, slot), validity); + self + } + + /// Resolve the validity of `(addr, slot)` with **slot ▸ account ▸ default**. + pub fn validity(&self, addr: Address, slot: U256) -> Validity { + if let Some(v) = self.slots.get(&(addr, slot)) { + return *v; + } + if let Some(v) = self.accounts.get(&addr) { + return *v; + } + self.default + } + + /// Whether `(addr, slot)` is currently volatile (subject to verification). + /// + /// `true` for [`Validity::Volatile`], and for [`Validity::ValidThrough`]`(m)` + /// once `now > m`. `false` for [`Validity::Pinned`] and a still-valid + /// `ValidThrough` (`now <= m`). + pub fn is_volatile(&self, addr: Address, slot: U256, now: u64) -> bool { + match self.validity(addr, slot) { + Validity::Pinned => false, + Validity::Volatile => true, + Validity::ValidThrough(m) => now > m, + } + } +} + +// --------------------------------------------------------------------------- +// 2. Clock +// --------------------------------------------------------------------------- + +/// Source of the current clock value used throughout the freshness model. +/// +/// Implementations return a monotone-ish `u64` in their own units. The two +/// built-ins are [`BlockClock`] (block number, the default) and [`WallClock`] +/// (unix seconds). +pub trait FreshnessClock: Send + Sync { + /// The current clock value (block number or unix seconds). + fn now(&self) -> u64; +} + +/// Block-number clock (the default). Cloning shares the underlying counter, so a +/// clone observed by a background task sees [`set_block`](Self::set_block) +/// updates made on the main thread. +#[derive(Clone, Debug, Default)] +pub struct BlockClock(Arc); + +impl BlockClock { + /// A block clock starting at block 0. + pub fn new() -> Self { + Self(Arc::new(AtomicU64::new(0))) + } + + /// A block clock starting at `block`. + pub fn at(block: u64) -> Self { + Self(Arc::new(AtomicU64::new(block))) + } + + /// Set the current block number. Shared across clones. + pub fn set_block(&self, block: u64) { + self.0.store(block, Ordering::Relaxed); + } +} + +impl FreshnessClock for BlockClock { + fn now(&self) -> u64 { + self.0.load(Ordering::Relaxed) + } +} + +/// Wall-clock clock: [`now`](FreshnessClock::now) returns unix seconds. +#[derive(Clone, Copy, Debug, Default)] +pub struct WallClock; + +impl FreshnessClock for WallClock { + fn now(&self) -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) + } +} + +// --------------------------------------------------------------------------- +// 3. Policy +// --------------------------------------------------------------------------- + +/// Decides which volatile candidate slots must be verified this cycle. +/// +/// The controller passes the volatile candidates (predicted read set) plus the +/// current observation stats and `now`; the policy returns the subset to +/// re-fetch. Correctness does not depend on the policy being complete — the +/// background validator always re-checks each sim's *actual* volatile read set +/// before trusting results — so a policy only trades RPC cost against how often a +/// `Corrected` verdict is needed. +pub trait FreshnessPolicy: Send { + /// Of these volatile candidate slots, which must be verified this cycle? + fn select( + &mut self, + candidates: &[(Address, U256)], + obs: &SlotObservationTracker, + now: u64, + ) -> Vec<(Address, U256)>; + + /// Hook called when the controller advances to a new block. + fn on_new_block(&mut self, _block: u64) {} +} + +/// Verifies every volatile candidate (safe / eager). Always correct, most RPC. +#[derive(Clone, Copy, Debug, Default)] +pub struct AlwaysVerify; + +impl FreshnessPolicy for AlwaysVerify { + fn select( + &mut self, + candidates: &[(Address, U256)], + _obs: &SlotObservationTracker, + _now: u64, + ) -> Vec<(Address, U256)> { + candidates.to_vec() + } +} + +/// Verifies nothing (trust-all). Selects no slots from the predicted set, though +/// the actual-read-set reconcile in the background validator can still surface +/// changes. +#[derive(Clone, Copy, Debug, Default)] +pub struct NeverVerify; + +impl FreshnessPolicy for NeverVerify { + fn select( + &mut self, + _candidates: &[(Address, U256)], + _obs: &SlotObservationTracker, + _now: u64, + ) -> Vec<(Address, U256)> { + Vec::new() + } +} + +/// Adaptive policy: verifies candidates the observation tracker flags via +/// [`should_refetch`](crate::cache::SlotObservationTracker::should_refetch). +#[derive(Clone, Debug, Default)] +pub struct ObservationDriven { + /// Thresholds for the underlying `should_refetch` heuristic. + pub params: FreshnessParams, +} + +impl ObservationDriven { + /// An observation-driven policy with the given parameters. + pub fn new(params: FreshnessParams) -> Self { + Self { params } + } +} + +impl FreshnessPolicy for ObservationDriven { + fn select( + &mut self, + candidates: &[(Address, U256)], + obs: &SlotObservationTracker, + now: u64, + ) -> Vec<(Address, U256)> { + candidates + .iter() + .copied() + .filter(|(addr, slot)| obs.should_refetch(*addr, *slot, now, &self.params)) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn addr(n: u8) -> Address { + Address::repeat_byte(n) + } + + // --- Classification ---------------------------------------------------- + + #[test] + fn registry_default_is_volatile() { + let reg = FreshnessRegistry::new(); + assert_eq!(reg.default_validity(), Validity::Volatile); + assert_eq!(reg.validity(addr(1), U256::from(0)), Validity::Volatile); + } + + #[test] + fn registry_with_default_overrides_unclassified() { + let reg = FreshnessRegistry::with_default(Validity::Pinned); + assert_eq!(reg.validity(addr(1), U256::from(0)), Validity::Pinned); + assert!(!reg.is_volatile(addr(1), U256::from(0), 100)); + } + + #[test] + fn registry_resolution_order_slot_account_default() { + let a = addr(1); + let mut reg = FreshnessRegistry::new(); // default Volatile + reg.pin(a); // account-level Pinned + reg.mark_volatile_slot(a, U256::from(7)); // slot-level Volatile + + // slot-level wins over account-level + assert_eq!(reg.validity(a, U256::from(7)), Validity::Volatile); + // account-level wins over default for a non-overridden slot + assert_eq!(reg.validity(a, U256::from(8)), Validity::Pinned); + // default for an unrelated account + assert_eq!(reg.validity(addr(2), U256::from(7)), Validity::Volatile); + } + + #[test] + fn is_volatile_per_variant() { + let a = addr(1); + let mut reg = FreshnessRegistry::new(); + reg.pin_slot(a, U256::from(1)); + reg.mark_volatile_slot(a, U256::from(2)); + reg.valid_through_slot(a, U256::from(3), 50); + + assert!(!reg.is_volatile(a, U256::from(1), 100)); // Pinned + assert!(reg.is_volatile(a, U256::from(2), 100)); // Volatile + } + + #[test] + fn valid_through_boundary() { + let a = addr(1); + let slot = U256::from(3); + let mut reg = FreshnessRegistry::new(); + reg.valid_through_slot(a, slot, 50); + + assert!(!reg.is_volatile(a, slot, 49)); // before + assert!(!reg.is_volatile(a, slot, 50)); // at boundary: still valid (now == m) + assert!(reg.is_volatile(a, slot, 51)); // after: now > m + } + + #[test] + fn registry_is_clone() { + let mut reg = FreshnessRegistry::new(); + reg.pin(addr(1)); + let clone = reg.clone(); + assert_eq!(clone.validity(addr(1), U256::from(0)), Validity::Pinned); + } + + // --- Clock ------------------------------------------------------------- + + #[test] + fn block_clock_default_and_set() { + let clock = BlockClock::new(); + assert_eq!(clock.now(), 0); + clock.set_block(123); + assert_eq!(clock.now(), 123); + } + + #[test] + fn block_clock_clone_shares_counter() { + let clock = BlockClock::at(10); + let clone = clock.clone(); + clock.set_block(42); + // The clone observes the update through the shared Arc. + assert_eq!(clone.now(), 42); + } + + #[test] + fn wall_clock_is_unix_seconds() { + let now = WallClock.now(); + // Sanity: after 2021-01-01. + assert!(now > 1_600_000_000); + } + + // --- Policy ------------------------------------------------------------ + + #[test] + fn always_verify_selects_all() { + let obs = SlotObservationTracker::new(); + let candidates = [(addr(1), U256::from(0)), (addr(2), U256::from(1))]; + let mut policy = AlwaysVerify; + assert_eq!(policy.select(&candidates, &obs, 0), candidates.to_vec()); + } + + #[test] + fn never_verify_selects_none() { + let obs = SlotObservationTracker::new(); + let candidates = [(addr(1), U256::from(0))]; + let mut policy = NeverVerify; + assert!(policy.select(&candidates, &obs, 0).is_empty()); + } + + #[test] + fn observation_driven_selects_only_should_refetch() { + let mut obs = SlotObservationTracker::new(); + let params = FreshnessParams::default(); + let stable = (addr(1), U256::from(0)); + let unknown = (addr(2), U256::from(0)); + + // Build a stable (never-changed) slot with enough observations so + // `should_refetch` returns false for it. + for now in 0..params.min_observations { + obs.observe(stable.0, stable.1, U256::from(42), now as u64); + } + let now = params.min_observations as u64 - 1; + assert!(!obs.should_refetch(stable.0, stable.1, now, ¶ms)); + assert!(obs.should_refetch(unknown.0, unknown.1, now, ¶ms)); + + let mut policy = ObservationDriven::new(params); + let selected = policy.select(&[stable, unknown], &obs, now); + assert_eq!(selected, vec![unknown]); + } +} From 71f8aa5389e3dfd72ce9dff84208d8ab14e1e5bd Mon Sep 17 00:00:00 2001 From: Kai Aldag Date: Mon, 15 Jun 2026 00:23:09 +0100 Subject: [PATCH 04/26] Phase 2 (step 3): verify_slots, purge_account, fetcher + overlay seams Add the EvmCache freshness primitives: - `verify_slots` re-fetches via the batch fetcher, compares to cached values, injects the changed ones, and returns `Vec`. - `purge_account` drops an account's info + storage from both the CacheDB overlay and the BlockchainDb accounts/storage maps. - `set_storage_batch_fetcher` (test/extensibility seam) and a `cached_storage_value` read helper. Add `SlotChange` to freshness, `EvmSnapshot::storage_value` and `EvmOverlay::override_slot` accessors for the background validator, plus a stub/failing fetcher helper in tests/common and the step-3 integration tests in tests/freshness.rs. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/cache/mod.rs | 124 +++++++++++++++++++ src/cache/overlay.rs | 13 ++ src/cache/snapshot.rs | 13 ++ src/freshness.rs | 21 ++++ tests/common/mod.rs | 33 ++++- tests/freshness.rs | 271 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 474 insertions(+), 1 deletion(-) create mode 100644 tests/freshness.rs diff --git a/src/cache/mod.rs b/src/cache/mod.rs index fab7faf..58b3b87 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -62,6 +62,7 @@ use tracing::{debug, instrument, trace, warn}; use crate::access_set::StorageAccessList; use crate::errors::{SimError, SimulationError, SimulationResult}; +use crate::freshness::SlotChange; use crate::inspector::TransferInspector; use bytecode::BytecodeCache; @@ -944,6 +945,129 @@ impl EvmCache { } } + /// Set (or replace) the batch storage fetcher. + /// + /// This is the seam the freshness controller and tests use to drive + /// re-verification without a live provider: a stubbed + /// [`StorageBatchFetchFn`] can be injected over a mocked-provider cache. + pub fn set_storage_batch_fetcher(&mut self, f: StorageBatchFetchFn) { + self.storage_batch_fetcher = Some(f); + } + + /// Return the currently-cached value for a storage slot, if any. + /// + /// Checks the CacheDB overlay (layer 1) first, then the BlockchainDb backend + /// (layer 2). Returns `None` when neither layer has seen the slot. Unlike + /// [`read_storage_slot`](Self::read_storage_slot) this never touches RPC. + pub fn cached_storage_value(&self, address: Address, slot: U256) -> Option { + if let Some(db_account) = self.db.cache.accounts.get(&address) + && let Some(value) = db_account.storage.get(&slot) + { + return Some(*value); + } + let storage = self.blockchain_db.storage().read(); + storage.get(&address).and_then(|s| s.get(&slot).copied()) + } + + /// Re-fetch the given slots via the batch fetcher, compare to the currently + /// cached values, and inject the ones that changed. + /// + /// For each slot whose freshly-fetched value differs from the cached value, + /// the fresh value is written into the cache via + /// [`inject_storage_batch`](Self::inject_storage_batch) and a [`SlotChange`] + /// is recorded. Slots that are unchanged, or that the fetcher fails to + /// return, are left as-is. Returns the set of changed slots. + /// + /// Requires a batch fetcher (set at construction or via + /// [`set_storage_batch_fetcher`](Self::set_storage_batch_fetcher)); errors if + /// none is available. This is the synchronous main-thread primitive; the + /// background validator performs the equivalent comparison against a snapshot. + pub fn verify_slots(&mut self, slots: &[(Address, U256)]) -> Result> { + if slots.is_empty() { + return Ok(Vec::new()); + } + let fetcher = self + .storage_batch_fetcher + .as_ref() + .ok_or_else(|| anyhow!("verify_slots requires a storage batch fetcher"))? + .clone(); + + // Snapshot the cached values before fetching so we compare against a + // stable baseline. + let cached: HashMap<(Address, U256), Option> = slots + .iter() + .map(|&(addr, slot)| ((addr, slot), self.cached_storage_value(addr, slot))) + .collect(); + + let results = (fetcher)(slots.to_vec()); + + let mut changed = Vec::new(); + let mut to_inject = Vec::new(); + for (addr, slot, fetched) in results { + let fresh = match fetched { + Ok(value) => value, + Err(e) => { + debug!(%addr, %slot, error = %e, "verify_slots: fetch failed, skipping slot"); + continue; + } + }; + // A slot the cache never saw is treated as old = ZERO (the value a + // sim would have read), so a non-zero fresh value counts as a change. + let old = cached + .get(&(addr, slot)) + .copied() + .flatten() + .unwrap_or(U256::ZERO); + if fresh != old { + to_inject.push((addr, slot, fresh)); + changed.push(SlotChange { + address: addr, + slot, + old, + new: fresh, + }); + } + } + + if !to_inject.is_empty() { + self.inject_storage_batch(&to_inject); + } + Ok(changed) + } + + /// Purge an account fully from both cache layers: its `AccountInfo` + /// (balance/nonce/code hash) **and** all of its storage. + /// + /// Removes `addr` from the CacheDB overlay accounts map, the BlockchainDb + /// accounts map, and the BlockchainDb storage map, so the next access + /// re-fetches a clean account from RPC. This is the account-level + /// counterpart to the storage-only [`purge_pool_storage`](Self::purge_pool_storage): + /// use it when an address is fully volatile (no pinned slots) and even its + /// balance/nonce/code can no longer be trusted. + pub fn purge_account(&mut self, addr: Address) { + // Layer 1: CacheDB overlay (accounts + their storage live together). + let overlay_removed = self.db.cache.accounts.remove(&addr).is_some(); + + // Layer 2: BlockchainDb accounts + storage maps. + let backend_account_removed = self + .blockchain_db + .accounts() + .write() + .remove(&addr) + .is_some(); + let backend_storage_removed = self.blockchain_db.storage().write().remove(&addr).is_some(); + + if overlay_removed || backend_account_removed || backend_storage_removed { + debug!( + account = %addr, + overlay_removed, + backend_account_removed, + backend_storage_removed, + "purged account from both cache layers" + ); + } + } + /// Get the chain ID used for EVM simulations. pub fn chain_id(&self) -> u64 { self.chain_id diff --git a/src/cache/overlay.rs b/src/cache/overlay.rs index c34201b..9af9514 100644 --- a/src/cache/overlay.rs +++ b/src/cache/overlay.rs @@ -331,6 +331,19 @@ impl EvmOverlay { evm.journaled_state.checkpoint_revert(checkpoint); Ok((result, access_list)) } + + /// Write a storage value into this overlay's dirty layer. + /// + /// The dirty layer takes precedence over the snapshot on subsequent reads, + /// so this lets a caller (e.g. the freshness validator) inject a fresh slot + /// value into a snapshot-backed overlay before re-running a simulation, + /// without mutating the shared snapshot. + pub fn override_slot(&mut self, address: Address, slot: U256, value: U256) { + self.dirty_storage + .entry(address) + .or_default() + .insert(slot, value); + } } impl revm::database_interface::DatabaseCommit for EvmOverlay { diff --git a/src/cache/snapshot.rs b/src/cache/snapshot.rs index 05de435..70dee8e 100644 --- a/src/cache/snapshot.rs +++ b/src/cache/snapshot.rs @@ -37,6 +37,19 @@ pub struct EvmSnapshot { pub(crate) spec_id: SpecId, } +impl EvmSnapshot { + /// Return the snapshot's value for a storage slot, if present. + /// + /// Used by the freshness validator to compare a freshly-fetched value + /// against the value the snapshot was built from. A missing entry means the + /// snapshot never captured that slot (it would read as zero in a sim). + pub fn storage_value(&self, address: Address, slot: U256) -> Option { + self.storage + .get(&address) + .and_then(|s| s.get(&slot).copied()) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/freshness.rs b/src/freshness.rs index d784a70..7238fec 100644 --- a/src/freshness.rs +++ b/src/freshness.rs @@ -404,6 +404,27 @@ impl FreshnessPolicy for ObservationDriven { } } +// --------------------------------------------------------------------------- +// 4. Results +// --------------------------------------------------------------------------- + +/// A storage slot whose freshly-fetched value differs from the cached value. +/// +/// Produced by [`EvmCache::verify_slots`](crate::cache::EvmCache::verify_slots) +/// and by the background validator; `old` is the value the snapshot/cache held, +/// `new` is the value the fetcher returned. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SlotChange { + /// Contract whose storage changed. + pub address: Address, + /// Storage slot key. + pub slot: U256, + /// Value previously held in the cache/snapshot. + pub old: U256, + /// Freshly-fetched value. + pub new: U256, +} + #[cfg(test)] mod tests { use super::*; diff --git a/tests/common/mod.rs b/tests/common/mod.rs index a82d270..cd638e0 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -5,6 +5,7 @@ //! ever reaches the network. #![allow(dead_code)] +use std::collections::HashMap; use std::sync::Arc; use alloy_primitives::{Address, Bytes, U256, hex}; @@ -14,7 +15,7 @@ use alloy_rpc_client::RpcClient; use alloy_sol_types::{SolCall, sol}; use alloy_transport::mock::Asserter; use anyhow::{Result, anyhow}; -use evm_fork_cache::cache::EvmCache; +use evm_fork_cache::cache::{EvmCache, StorageBatchFetchFn}; use revm::context::result::ExecutionResult; use revm::state::{AccountInfo, Bytecode}; @@ -94,6 +95,36 @@ pub fn balance_of(cache: &mut EvmCache, token: Address, owner: Address) -> Resul } } +/// Build a stub [`StorageBatchFetchFn`] that returns chosen "current" values. +/// +/// `values` maps `(address, slot)` to the value the fetcher reports. Any +/// requested slot not present in the map is reported as `U256::ZERO` (matching +/// how an unseen slot reads in a simulation). This is the offline stand-in for +/// the real RPC batch fetcher. +pub fn stub_fetcher(values: HashMap<(Address, U256), U256>) -> StorageBatchFetchFn { + Arc::new(move |requests: Vec<(Address, U256)>| { + requests + .into_iter() + .map(|(addr, slot)| { + let value = values.get(&(addr, slot)).copied().unwrap_or(U256::ZERO); + (addr, slot, Ok(value)) + }) + .collect() + }) +} + +/// Build a stub [`StorageBatchFetchFn`] that fails every request. +/// +/// Used to exercise the `Unverified` / error paths offline. +pub fn failing_fetcher() -> StorageBatchFetchFn { + Arc::new(|requests: Vec<(Address, U256)>| { + requests + .into_iter() + .map(|(addr, slot)| (addr, slot, Err(anyhow!("stub fetcher error")))) + .collect() + }) +} + /// Submit a `transfer(to, amount)` to a `MockERC20`, committing the state change. pub fn transfer( cache: &mut EvmCache, diff --git a/tests/freshness.rs b/tests/freshness.rs new file mode 100644 index 0000000..38d060d --- /dev/null +++ b/tests/freshness.rs @@ -0,0 +1,271 @@ +//! Offline integration tests for the Phase 2 freshness primitives and the +//! optimistic verify-and-rerun loop. +//! +//! Everything runs fully offline: the cache is built over a mocked provider and +//! all "current" on-chain values come from a stubbed [`StorageBatchFetchFn`] +//! injected via `set_storage_batch_fetcher`, so no test reaches the network. + +mod common; + +use std::collections::HashMap; +use std::sync::Arc; + +use alloy_primitives::{Address, Bytes, U256}; +use alloy_sol_types::SolCall; +use anyhow::Result; + +use common::{ + MOCK_ERC20_BALANCE_SLOT, failing_fetcher, install_default_account, install_mock_erc20, + setup_cache, stub_fetcher, +}; +use evm_fork_cache::cache::{EvmCache, EvmOverlay}; + +// --------------------------------------------------------------------------- +// EvmCache::verify_slots +// --------------------------------------------------------------------------- + +#[tokio::test(flavor = "multi_thread")] +async fn verify_slots_detects_and_injects_changes() -> Result<()> { + let mut cache = setup_cache().await?; + let contract = Address::repeat_byte(0x11); + install_mock_erc20(&mut cache, contract); + + let slot_a = U256::from(10); + let slot_b = U256::from(20); + // Cache holds these values. + cache.inject_storage_batch(&[ + (contract, slot_a, U256::from(100)), + (contract, slot_b, U256::from(200)), + ]); + + // Stub reports slot_a changed, slot_b unchanged. + let values = HashMap::from([ + ((contract, slot_a), U256::from(999)), + ((contract, slot_b), U256::from(200)), + ]); + cache.set_storage_batch_fetcher(stub_fetcher(values)); + + let changed = cache.verify_slots(&[(contract, slot_a), (contract, slot_b)])?; + + assert_eq!(changed.len(), 1, "only slot_a changed"); + let change = &changed[0]; + assert_eq!(change.address, contract); + assert_eq!(change.slot, slot_a); + assert_eq!(change.old, U256::from(100)); + assert_eq!(change.new, U256::from(999)); + + // The fresh value was injected; the unchanged one is untouched. + assert_eq!( + cache.cached_storage_value(contract, slot_a), + Some(U256::from(999)) + ); + assert_eq!( + cache.cached_storage_value(contract, slot_b), + Some(U256::from(200)) + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn verify_slots_unchanged_returns_empty() -> Result<()> { + let mut cache = setup_cache().await?; + let contract = Address::repeat_byte(0x22); + install_mock_erc20(&mut cache, contract); + + let slot = U256::from(7); + cache.inject_storage_batch(&[(contract, slot, U256::from(42))]); + cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([( + (contract, slot), + U256::from(42), + )]))); + + let changed = cache.verify_slots(&[(contract, slot)])?; + assert!(changed.is_empty(), "no change should be reported"); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn verify_slots_treats_unseen_slot_as_zero() -> Result<()> { + let mut cache = setup_cache().await?; + let contract = Address::repeat_byte(0x33); + install_mock_erc20(&mut cache, contract); + + // Slot never cached; fetcher reports a non-zero value → counts as a change + // from the implicit zero a sim would have read. + let slot = U256::from(5); + cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([( + (contract, slot), + U256::from(77), + )]))); + + let changed = cache.verify_slots(&[(contract, slot)])?; + assert_eq!(changed.len(), 1); + assert_eq!(changed[0].old, U256::ZERO); + assert_eq!(changed[0].new, U256::from(77)); + assert_eq!( + cache.cached_storage_value(contract, slot), + Some(U256::from(77)) + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn verify_slots_skips_failed_fetches() -> Result<()> { + let mut cache = setup_cache().await?; + // A fetcher that errors every request: failed fetches are skipped (not + // treated as changes), so verify_slots returns no changes and does not panic. + cache.set_storage_batch_fetcher(failing_fetcher()); + let contract = Address::repeat_byte(0x44); + cache.inject_storage_batch(&[(contract, U256::from(1), U256::from(5))]); + let changed = cache.verify_slots(&[(contract, U256::from(1))])?; + assert!( + changed.is_empty(), + "failed fetches are skipped, not changes" + ); + // Cached value is unchanged. + assert_eq!( + cache.cached_storage_value(contract, U256::from(1)), + Some(U256::from(5)) + ); + + Ok(()) +} + +// --------------------------------------------------------------------------- +// EvmCache::purge_account +// --------------------------------------------------------------------------- + +#[tokio::test(flavor = "multi_thread")] +async fn purge_account_drops_account_and_storage_from_both_layers() -> Result<()> { + let mut cache = setup_cache().await?; + let token = Address::repeat_byte(0x55); + let owner = Address::repeat_byte(0x66); + + install_default_account(&mut cache, Address::ZERO); + install_default_account(&mut cache, owner); + install_mock_erc20(&mut cache, token); + + // Populate the CacheDB overlay (layer 1) via an EVM read and the + // BlockchainDb backend (layer 2) directly. + cache.insert_mapping_storage_slot( + token, + U256::from(MOCK_ERC20_BALANCE_SLOT), + owner, + U256::from(1000), + )?; + let _ = common::balance_of(&mut cache, token, owner)?; + assert!( + cache.cache_db_storage_slot_count(token) > 0, + "overlay populated" + ); + + cache.inject_storage_batch(&[(token, U256::from(99), U256::from(1))]); + assert!( + cache.pool_storage_slot_count(token) > 0, + "backend populated" + ); + + // The account info exists in the overlay (from the EVM read / install). + assert!( + cache.db_mut().cache.accounts.contains_key(&token), + "overlay account present before purge" + ); + + cache.purge_account(token); + + // Account gone from the overlay accounts map (which also holds its storage). + assert!( + !cache.db_mut().cache.accounts.contains_key(&token), + "overlay account removed" + ); + assert_eq!( + cache.cache_db_storage_slot_count(token), + 0, + "overlay storage gone" + ); + // Storage gone from the backend. + assert_eq!( + cache.pool_storage_slot_count(token), + 0, + "backend storage gone" + ); + // Account gone from the backend accounts map. + { + let accounts = cache.blockchain_db().accounts().read(); + assert!(!accounts.contains_key(&token), "backend account removed"); + } + + Ok(()) +} + +// --------------------------------------------------------------------------- +// EvmOverlay::call_raw_with_access_list (read-set capture) +// --------------------------------------------------------------------------- + +#[tokio::test(flavor = "multi_thread")] +async fn overlay_call_raw_with_access_list_captures_read_set() -> Result<()> { + let mut cache = setup_cache().await?; + let token = Address::repeat_byte(0x77); + let owner = Address::repeat_byte(0x88); + + install_default_account(&mut cache, Address::ZERO); + install_default_account(&mut cache, owner); + install_mock_erc20(&mut cache, token); + let balance_slot = U256::from(MOCK_ERC20_BALANCE_SLOT); + cache.insert_mapping_storage_slot(token, balance_slot, owner, U256::from(1000))?; + + let snapshot = cache.create_snapshot(); + let mut overlay = EvmOverlay::new(Arc::clone(&snapshot), None); + + // balanceOf(owner) reads the token's balance mapping slot. + let call = common::MockERC20::balanceOfCall { account: owner }; + let (result, access) = + overlay.call_raw_with_access_list(owner, token, Bytes::from(call.abi_encode()))?; + + assert!(result.is_success(), "balanceOf should succeed: {result:?}"); + assert!(access.accounts.contains(&token), "token account touched"); + // The hashed balance slot for owner should be in the read set. + let hashed = { + use alloy_sol_types::SolValue; + let key = alloy_primitives::keccak256((owner, balance_slot).abi_encode()); + U256::from_be_bytes(key.0) + }; + assert!( + access.slots.contains(&(token, hashed)), + "balance mapping slot captured in read set" + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn overlay_override_slot_takes_precedence() -> Result<()> { + let mut cache = setup_cache().await?; + let contract = Address::repeat_byte(0x99); + install_mock_erc20(&mut cache, contract); + let slot = U256::from(3); + cache.inject_storage_batch(&[(contract, slot, U256::from(1))]); + + let snapshot = cache.create_snapshot(); + let mut overlay = EvmOverlay::new(snapshot, None); + overlay.override_slot(contract, slot, U256::from(999)); + + use revm::database_interface::Database; + assert_eq!(overlay.storage(contract, slot)?, U256::from(999)); + + Ok(()) +} + +// Compile-time guard: a cache built over a mocked provider exposes a fetcher. +#[tokio::test(flavor = "multi_thread")] +async fn cache_has_fetcher_over_mock_provider() -> Result<()> { + let cache: EvmCache = setup_cache().await?; + assert!( + cache.storage_batch_fetcher().is_some(), + "mock-provider cache has a fetcher" + ); + Ok(()) +} From 472502098a115767a2e064418f609b3dfe27b86c Mon Sep 17 00:00:00 2001 From: Kai Aldag Date: Mon, 15 Jun 2026 00:30:10 +0100 Subject: [PATCH 05/26] Phase 2 (step 4): FreshnessController + SpeculativeSim optimistic loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the optimistic verify-and-rerun loop: - `SimRequest`, `Validation` (Confirmed/Corrected/Unverified), `SpeculativeSim` (optimistic results + deferred-validation JoinHandle, abort-on-drop), and the generic `FreshnessController`. - `run` drains pending corrections, snapshots, runs optimistic sims capturing per-sim volatile read sets, asks the policy which predicted candidates to verify, then spawns a Send-only background validator and returns immediately. - The validator re-fetches (policy set ∪ actual read sets), compares to the snapshot, observes into the shared tracker, and on a mismatch queues corrections + re-runs only the affected sims (overlay override_slot) → Corrected; fetcher error → Unverified. The full-loop integration tests cover the match/mismatch/unverified paths, pending flow-back, selective re-run, NeverVerify reconcile, a pinned slot, ValidThrough boundary, and BlockClock vs WallClock. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/freshness.rs | 460 ++++++++++++++++++++++++++++++++++++++++++++- tests/freshness.rs | 422 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 876 insertions(+), 6 deletions(-) diff --git a/src/freshness.rs b/src/freshness.rs index 7238fec..aa74994 100644 --- a/src/freshness.rs +++ b/src/freshness.rs @@ -51,14 +51,20 @@ //! assert!(NeverVerify.select(&candidates, &obs, now).is_empty()); //! ``` -use std::collections::HashMap; -use std::sync::Arc; +use std::collections::{HashMap, HashSet}; use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; use std::time::{SystemTime, UNIX_EPOCH}; -use alloy_primitives::{Address, U256}; +use alloy_eips::eip2930::AccessList; +use alloy_primitives::{Address, Bytes, U256}; +use revm::context::result::ExecutionResult; +use tokio::task::JoinHandle; -use crate::cache::SlotObservationTracker; +use crate::cache::{ + CallSimulationResult, EvmCache, EvmOverlay, EvmSnapshot, SlotObservationTracker, + StorageBatchFetchFn, TxConfig, +}; /// Default minimum observations before the change-frequency data is trusted. pub const DEFAULT_MIN_OBSERVATIONS: u32 = 10; @@ -425,6 +431,452 @@ pub struct SlotChange { pub new: U256, } +/// The deferred verdict on a [`SpeculativeSim`]'s optimistic results. +pub enum Validation { + /// Nothing the sims read had changed; the optimistic results are correct. + Confirmed, + /// At least one read slot changed. `results` is the optimistic set with the + /// affected sims re-run against the fresh values; `changed` lists the slots + /// that differed (also queued for flow-back into the cache). + Corrected { + /// Optimistic results with the affected sims replaced by re-runs. + results: Vec, + /// Slots whose fresh value differed from the snapshot. + changed: Vec, + }, + /// The fetcher failed, so the results could not be validated. The optimistic + /// results are *not* trusted. + Unverified { + /// Human-readable description of why validation could not complete. + reason: String, + }, +} + +impl std::fmt::Debug for Validation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Validation::Confirmed => write!(f, "Confirmed"), + Validation::Corrected { changed, .. } => f + .debug_struct("Corrected") + .field("changed", changed) + .finish_non_exhaustive(), + Validation::Unverified { reason } => f + .debug_struct("Unverified") + .field("reason", reason) + .finish(), + } + } +} + +/// A single non-committing simulation request for the optimistic loop. +/// +/// `tx.access_list` is the *predicted* read set (a performance hint that seeds +/// the verify candidates); correctness does not depend on it because the +/// background validator re-checks each sim's actual volatile read set. +#[derive(Clone, Debug)] +pub struct SimRequest { + /// Transaction sender. + pub from: Address, + /// Call target. + pub to: Address, + /// Calldata. + pub calldata: Bytes, + /// Per-call tx environment; `tx.access_list` is the predicted read set. + pub tx: TxConfig, +} + +impl SimRequest { + /// A zero-value request with default tx environment. + pub fn new(from: Address, to: Address, calldata: Bytes) -> Self { + Self { + from, + to, + calldata, + tx: TxConfig::default(), + } + } + + /// Set the predicted read set (EIP-2930 access list hint). + pub fn with_access_list(mut self, access_list: AccessList) -> Self { + self.tx.access_list = Some(access_list); + self + } +} + +/// Optimistic simulation results plus a handle to their deferred validation. +/// +/// Returned by [`FreshnessController::run`] as soon as the optimistic sims +/// finish (without awaiting RPC). Read [`optimistic`](Self::optimistic) +/// immediately, then `await` [`validate`](Self::validate) for the verdict. +/// Dropping the handle aborts the background validation task. +pub struct SpeculativeSim { + optimistic: Vec, + /// `Option` so `validate`/`into_optimistic` can take the handle and skip the + /// abort-on-drop; `Drop` only aborts a handle still left in place. + validation: Option>, +} + +impl SpeculativeSim { + /// The optimistic results, readable before validation completes. + pub fn optimistic(&self) -> &[CallSimulationResult] { + &self.optimistic + } + + /// Consume the handle and return the optimistic results, aborting the + /// background validation task. + pub fn into_optimistic(mut self) -> Vec { + if let Some(handle) = self.validation.take() { + handle.abort(); + } + std::mem::take(&mut self.optimistic) + } + + /// Await the deferred validation verdict. + /// + /// If the background task panicked or was cancelled, returns + /// [`Validation::Unverified`]. + pub async fn validate(mut self) -> Validation { + let handle = self + .validation + .take() + .expect("validation handle taken twice"); + match handle.await { + Ok(v) => v, + Err(e) => Validation::Unverified { + reason: format!("validation task failed: {e}"), + }, + } + } +} + +impl Drop for SpeculativeSim { + fn drop(&mut self) { + if let Some(handle) = self.validation.take() { + handle.abort(); + } + } +} + +// --------------------------------------------------------------------------- +// Controller +// --------------------------------------------------------------------------- + +/// Drives the optimistic verify-and-rerun loop over an [`EvmCache`]. +/// +/// Holds the freshness [`FreshnessRegistry`], the shared +/// [`SlotObservationTracker`], a [`FreshnessPolicy`], a [`FreshnessClock`], the +/// [`FreshnessParams`], and the pending-corrections queue. The tracker and the +/// pending queue are `Arc>` so the background validator can update them +/// without touching the `!Send` cache. +/// +/// # Runtime requirement +/// [`run`](Self::run) spawns a background task and the (synchronous) fetcher uses +/// `block_in_place` internally, so a **multi-thread** tokio runtime is required +/// (`#[tokio::main(flavor = "multi_thread")]` or +/// `Builder::new_multi_thread()`), mirroring the [`EvmCache`] constructor note. +pub struct FreshnessController { + registry: FreshnessRegistry, + tracker: Arc>, + policy: P, + clock: C, + params: FreshnessParams, + pending: Arc>>, +} + +impl FreshnessController { + /// Build a controller with the default [`BlockClock`]. + pub fn new(registry: FreshnessRegistry, policy: P) -> Self { + Self::with_clock(registry, policy, BlockClock::new()) + } +} + +impl FreshnessController { + /// Build a controller with an explicit clock. + pub fn with_clock(registry: FreshnessRegistry, policy: P, clock: C) -> Self { + Self { + registry, + tracker: Arc::new(Mutex::new(SlotObservationTracker::new())), + policy, + clock, + params: FreshnessParams::default(), + pending: Arc::new(Mutex::new(Vec::new())), + } + } + + /// Replace the [`FreshnessParams`] used by the observation tracker. + pub fn with_params(mut self, params: FreshnessParams) -> Self { + self.params = params; + self + } + + /// Use an existing shared observation tracker (e.g. a persisted one). + pub fn with_tracker(mut self, tracker: Arc>) -> Self { + self.tracker = tracker; + self + } + + /// The shared observation tracker. + pub fn tracker(&self) -> &Arc> { + &self.tracker + } + + /// The freshness registry. + pub fn registry(&self) -> &FreshnessRegistry { + &self.registry + } + + /// Mutable access to the freshness registry. + pub fn registry_mut(&mut self) -> &mut FreshnessRegistry { + &mut self.registry + } + + /// Number of corrections waiting to be drained into the cache on the next + /// [`run`](Self::run). + pub fn pending_len(&self) -> usize { + self.pending.lock().unwrap().len() + } + + /// Advance to a new block: bump a [`BlockClock`] is the caller's job (the + /// clock is shared); this notifies the policy. + pub fn on_new_block(&mut self, block: u64) { + self.policy.on_new_block(block); + } + + /// Run the optimistic loop for a batch of requests. + /// + /// 1. Drain queued corrections from prior cycles into the cache. + /// 2. Snapshot the cache and grab the batch fetcher. + /// 3. Run each request optimistically against the snapshot, capturing its + /// actual volatile read set. + /// 4. Compute the predicted volatile candidates and ask the policy which to + /// verify. + /// 5. Spawn the background validator (Send data only) and return a + /// [`SpeculativeSim`] immediately. + pub fn run( + &mut self, + cache: &mut EvmCache, + requests: Vec, + ) -> anyhow::Result { + let now = self.clock.now(); + + // 1. Drain pending corrections into the cache before snapshotting. + { + let mut pending = self.pending.lock().unwrap(); + if !pending.is_empty() { + let injects: Vec<(Address, U256, U256)> = + pending.iter().map(|c| (c.address, c.slot, c.new)).collect(); + cache.inject_storage_batch(&injects); + pending.clear(); + } + } + + // 2. Snapshot + fetcher (Arc clones, both Send). + let snapshot = cache.create_snapshot(); + let fetcher = cache.storage_batch_fetcher().cloned(); + + // 3. Optimistic sims + per-sim actual volatile read sets. + let mut optimistic = Vec::with_capacity(requests.len()); + let mut read_sets: Vec> = Vec::with_capacity(requests.len()); + for req in &requests { + let mut overlay = EvmOverlay::new(Arc::clone(&snapshot), None); + let (result, access) = + overlay.call_raw_with_access_list(req.from, req.to, req.calldata.clone())?; + optimistic.push(result_to_sim(result, &access.to_eip2930())); + + let volatile: Vec<(Address, U256)> = access + .slots + .iter() + .copied() + .filter(|(addr, slot)| self.registry.is_volatile(*addr, *slot, now)) + .collect(); + read_sets.push(volatile); + } + + // 4. Predicted candidates (union of request access lists, volatile only). + let mut candidate_set: HashSet<(Address, U256)> = HashSet::new(); + for req in &requests { + if let Some(al) = &req.tx.access_list { + for item in &al.0 { + for key in &item.storage_keys { + let slot = U256::from_be_bytes(key.0); + if self.registry.is_volatile(item.address, slot, now) { + candidate_set.insert((item.address, slot)); + } + } + } + } + } + let candidates: Vec<(Address, U256)> = candidate_set.into_iter().collect(); + let verify_set = { + let tracker = self.tracker.lock().unwrap(); + self.policy.select(&candidates, &tracker, now) + }; + + // 5. Spawn the validator with Send-only data. + let registry = self.registry.clone(); + let tracker = Arc::clone(&self.tracker); + let pending = Arc::clone(&self.pending); + let optimistic_for_task = optimistic.clone(); + let validation = tokio::spawn(async move { + run_validator(ValidatorInput { + snapshot, + fetcher, + requests, + read_sets, + registry, + tracker, + pending, + now, + verify_set, + optimistic: optimistic_for_task, + }) + }); + + Ok(SpeculativeSim { + optimistic, + validation: Some(validation), + }) + } +} + +/// Owned inputs handed to the background validator (all `Send`). +struct ValidatorInput { + snapshot: Arc, + fetcher: Option, + requests: Vec, + read_sets: Vec>, + registry: FreshnessRegistry, + tracker: Arc>, + pending: Arc>>, + now: u64, + verify_set: Vec<(Address, U256)>, + optimistic: Vec, +} + +/// The background validation routine. Touches only `Send` data — never the cache. +fn run_validator(input: ValidatorInput) -> Validation { + let ValidatorInput { + snapshot, + fetcher, + requests, + read_sets, + registry, + tracker, + pending, + now, + verify_set, + optimistic, + } = input; + + let Some(fetcher) = fetcher else { + return Validation::Unverified { + reason: "no storage batch fetcher available".to_string(), + }; + }; + + // verify = policy-selected set ∪ each sim's actual volatile read set, + // re-filtered through the registry clone so only currently-volatile slots + // are checked (defensive: read sets and the policy selection are already + // volatile-filtered on the main thread). + let mut verify: HashSet<(Address, U256)> = verify_set.into_iter().collect(); + for set in &read_sets { + verify.extend(set.iter().copied()); + } + verify.retain(|(addr, slot)| registry.is_volatile(*addr, *slot, now)); + if verify.is_empty() { + return Validation::Confirmed; + } + let verify: Vec<(Address, U256)> = verify.into_iter().collect(); + + // Fetch fresh values. Any error → Unverified (never trust silently). + let results = (fetcher)(verify.clone()); + let mut fresh: HashMap<(Address, U256), U256> = HashMap::new(); + for (addr, slot, value) in results { + match value { + Ok(v) => { + fresh.insert((addr, slot), v); + } + Err(e) => { + return Validation::Unverified { + reason: format!("fetch failed for {addr}:{slot}: {e}"), + }; + } + } + } + + // Compare against the snapshot, observe each checked slot, collect changes. + let mut changed = Vec::new(); + { + let mut tracker = tracker.lock().unwrap(); + for &(addr, slot) in &verify { + let new = fresh.get(&(addr, slot)).copied().unwrap_or(U256::ZERO); + let old = snapshot.storage_value(addr, slot).unwrap_or(U256::ZERO); + tracker.observe(addr, slot, new, now); + if new != old { + changed.push(SlotChange { + address: addr, + slot, + old, + new, + }); + } + } + } + + if changed.is_empty() { + return Validation::Confirmed; + } + + // Queue corrections for flow-back into the cache on the next run. + { + let mut pending = pending.lock().unwrap(); + pending.extend(changed.iter().cloned()); + } + + // Re-run only the sims whose read set intersects the changed slots. + let changed_keys: HashSet<(Address, U256)> = + changed.iter().map(|c| (c.address, c.slot)).collect(); + let overrides: Vec<(Address, U256, U256)> = + changed.iter().map(|c| (c.address, c.slot, c.new)).collect(); + + let mut results = optimistic; + for (i, req) in requests.iter().enumerate() { + let read_set = &read_sets[i]; + let intersects = read_set.iter().any(|k| changed_keys.contains(k)); + if !intersects { + continue; + } + let mut overlay = EvmOverlay::new(Arc::clone(&snapshot), None); + for &(addr, slot, value) in &overrides { + overlay.override_slot(addr, slot, value); + } + if let Ok((result, access)) = + overlay.call_raw_with_access_list(req.from, req.to, req.calldata.clone()) + { + results[i] = result_to_sim(result, &access.to_eip2930()); + } + } + + Validation::Corrected { results, changed } +} + +/// Build a [`CallSimulationResult`] from a non-committing execution result and +/// its captured access list. `token_deltas` is empty (the optimistic path does +/// not run transfer tracking); gas and logs come from the execution result. +fn result_to_sim(result: ExecutionResult, access_list: &AccessList) -> CallSimulationResult { + let (gas_used, logs) = match &result { + ExecutionResult::Success { gas_used, logs, .. } => (*gas_used, logs.clone()), + ExecutionResult::Revert { gas_used, .. } => (*gas_used, Vec::new()), + ExecutionResult::Halt { gas_used, .. } => (*gas_used, Vec::new()), + }; + CallSimulationResult { + gas_used, + token_deltas: HashMap::new(), + logs, + access_list: access_list.clone(), + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/tests/freshness.rs b/tests/freshness.rs index 38d060d..030e879 100644 --- a/tests/freshness.rs +++ b/tests/freshness.rs @@ -15,10 +15,27 @@ use alloy_sol_types::SolCall; use anyhow::Result; use common::{ - MOCK_ERC20_BALANCE_SLOT, failing_fetcher, install_default_account, install_mock_erc20, - setup_cache, stub_fetcher, + MOCK_ERC20_BALANCE_SLOT, MockERC20, failing_fetcher, install_default_account, + install_mock_erc20, setup_cache, stub_fetcher, }; use evm_fork_cache::cache::{EvmCache, EvmOverlay}; +use evm_fork_cache::freshness::{ + AlwaysVerify, FreshnessController, FreshnessRegistry, NeverVerify, SimRequest, Validation, + WallClock, +}; + +/// Hashed storage slot of `balanceOf[owner]` for the MockERC20 fixture. +fn balance_slot_for(owner: Address) -> U256 { + use alloy_sol_types::SolValue; + let key = + alloy_primitives::keccak256((owner, U256::from(MOCK_ERC20_BALANCE_SLOT)).abi_encode()); + U256::from_be_bytes(key.0) +} + +/// Encode a `transfer(to, amount)` call. +fn transfer_calldata(to: Address, amount: U256) -> Bytes { + Bytes::from(MockERC20::transferCall { to, amount }.abi_encode()) +} // --------------------------------------------------------------------------- // EvmCache::verify_slots @@ -269,3 +286,404 @@ async fn cache_has_fetcher_over_mock_provider() -> Result<()> { ); Ok(()) } + +// --------------------------------------------------------------------------- +// FreshnessController::run — the optimistic loop +// --------------------------------------------------------------------------- + +/// Build a cache with a MockERC20 whose `owner` balance is `balance`. +async fn cache_with_balance(token: Address, owner: Address, balance: U256) -> Result { + let mut cache = setup_cache().await?; + install_default_account(&mut cache, Address::ZERO); + install_default_account(&mut cache, owner); + install_mock_erc20(&mut cache, token); + if balance > U256::ZERO { + cache.inject_storage_batch(&[(token, balance_slot_for(owner), balance)]); + } + Ok(cache) +} + +#[tokio::test(flavor = "multi_thread")] +async fn run_match_path_confirmed() -> Result<()> { + let token = Address::repeat_byte(0x11); + let owner = Address::repeat_byte(0x22); + let recipient = Address::repeat_byte(0x33); + + // Owner funded; the optimistic transfer succeeds. + let mut cache = cache_with_balance(token, owner, U256::from(1000)).await?; + // Fetcher reports the SAME balance → nothing changed. + cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([( + (token, balance_slot_for(owner)), + U256::from(1000), + )]))); + + let mut controller = FreshnessController::new(FreshnessRegistry::new(), AlwaysVerify); + let req = SimRequest::new(owner, token, transfer_calldata(recipient, U256::from(100))); + let sim = controller.run(&mut cache, vec![req])?; + + // optimistic() is readable before validate(). + assert_eq!(sim.optimistic().len(), 1); + let optimistic_gas = sim.optimistic()[0].gas_used; + assert!(optimistic_gas > 0); + + let validation = sim.validate().await; + assert!( + matches!(validation, Validation::Confirmed), + "unchanged values should confirm: {validation:?}" + ); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn run_mismatch_path_corrected_only_affected_rerun() -> Result<()> { + let token = Address::repeat_byte(0x44); + let owner = Address::repeat_byte(0x55); + let recipient = Address::repeat_byte(0x66); + + // A second, independent token whose slot will NOT change. + let token2 = Address::repeat_byte(0x77); + let owner2 = Address::repeat_byte(0x88); + + // Both owners are funded so the optimistic transfers SUCCEED (and so their + // balance slots land in the captured read set). The captured read set is the + // basis for reconciliation; a reverting sim records no SLOADs. + let mut cache = setup_cache().await?; + install_default_account(&mut cache, Address::ZERO); + install_default_account(&mut cache, owner); + install_default_account(&mut cache, owner2); + install_mock_erc20(&mut cache, token); + install_mock_erc20(&mut cache, token2); + cache.inject_storage_batch(&[ + (token, balance_slot_for(owner), U256::from(1000)), + (token2, balance_slot_for(owner2), U256::from(5000)), + ]); + + // Fetcher: owner's balance slot DROPPED to 50 (< the 100 transfer, so the + // re-run now reverts); owner2's slot unchanged; recipient slots read as zero + // (matching the snapshot → no change). + cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([ + ((token, balance_slot_for(owner)), U256::from(50)), + ((token2, balance_slot_for(owner2)), U256::from(5000)), + ]))); + + let mut controller = FreshnessController::new(FreshnessRegistry::new(), AlwaysVerify); + let req1 = SimRequest::new(owner, token, transfer_calldata(recipient, U256::from(100))); + let req2 = SimRequest::new( + owner2, + token2, + transfer_calldata(recipient, U256::from(100)), + ); + let sim = controller.run(&mut cache, vec![req1, req2])?; + + // Optimistic: both transfers succeeded (each emits a Transfer log). + let opt = sim.optimistic().to_vec(); + assert_eq!(opt.len(), 2); + assert!( + !opt[0].logs.is_empty(), + "req1 optimistic should succeed (a log)" + ); + assert!( + !opt[1].logs.is_empty(), + "req2 optimistic should succeed (a log)" + ); + + let validation = sim.validate().await; + match validation { + Validation::Corrected { results, changed } => { + // Exactly owner's balance slot changed. + assert_eq!( + changed.len(), + 1, + "only owner's balance changed: {changed:?}" + ); + assert_eq!(changed[0].address, token); + assert_eq!(changed[0].slot, balance_slot_for(owner)); + assert_eq!(changed[0].old, U256::from(1000)); + assert_eq!(changed[0].new, U256::from(50)); + + // req1 was re-run with the reduced balance → now reverts (no log) and + // differs from its optimistic (successful) result. + assert!( + results[0].logs.is_empty(), + "corrected req1 should now revert and emit no log" + ); + assert_ne!( + results[0].gas_used, opt[0].gas_used, + "corrected req1 gas should differ from the optimistic success" + ); + + // req2's slot did not change → its result is untouched (== optimistic). + assert_eq!(results[1].gas_used, opt[1].gas_used, "req2 not re-run"); + assert_eq!(results[1].logs.len(), opt[1].logs.len(), "req2 unchanged"); + } + other => panic!("expected Corrected, got {other:?}"), + } + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn run_drains_pending_on_next_run() -> Result<()> { + let token = Address::repeat_byte(0x11); + let owner = Address::repeat_byte(0x22); + let recipient = Address::repeat_byte(0x33); + + // Owner funded with 1000 so the optimistic transfer succeeds (read set + // captures the balance slot). Fetcher reports a CHANGED balance of 2000. + let mut cache = setup_cache().await?; + install_default_account(&mut cache, Address::ZERO); + install_default_account(&mut cache, owner); + install_mock_erc20(&mut cache, token); + cache.inject_storage_batch(&[(token, balance_slot_for(owner), U256::from(1000))]); + cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([( + (token, balance_slot_for(owner)), + U256::from(2000), + )]))); + + let mut controller = FreshnessController::new(FreshnessRegistry::new(), AlwaysVerify); + + // First run: detects the change and queues a correction. + let sim = controller.run( + &mut cache, + vec![SimRequest::new( + owner, + token, + transfer_calldata(recipient, U256::from(100)), + )], + )?; + let validation = sim.validate().await; + assert!(matches!(validation, Validation::Corrected { .. })); + assert_eq!(controller.pending_len(), 1, "a correction was queued"); + + // The live cache still holds the OLD value (no cross-thread mutation). + assert_eq!( + cache.cached_storage_value(token, balance_slot_for(owner)), + Some(U256::from(1000)) + ); + + // Second run: drains the pending correction into the cache before snapshotting. + let sim = controller.run( + &mut cache, + vec![SimRequest::new( + owner, + token, + transfer_calldata(recipient, U256::from(100)), + )], + )?; + assert_eq!(controller.pending_len(), 0, "pending drained"); + assert_eq!( + cache.cached_storage_value(token, balance_slot_for(owner)), + Some(U256::from(2000)), + "correction applied to the live cache" + ); + + // The optimistic transfer still succeeds and the fetcher now matches the + // applied value → Confirmed. + assert!( + !sim.optimistic()[0].logs.is_empty(), + "optimistic still succeeds" + ); + let validation = sim.validate().await; + assert!( + matches!(validation, Validation::Confirmed), + "{validation:?}" + ); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn run_unverified_on_fetcher_error() -> Result<()> { + let token = Address::repeat_byte(0x44); + let owner = Address::repeat_byte(0x55); + let recipient = Address::repeat_byte(0x66); + + let mut cache = cache_with_balance(token, owner, U256::from(1000)).await?; + cache.set_storage_batch_fetcher(failing_fetcher()); + + let mut controller = FreshnessController::new(FreshnessRegistry::new(), AlwaysVerify); + let sim = controller.run( + &mut cache, + vec![SimRequest::new( + owner, + token, + transfer_calldata(recipient, U256::from(100)), + )], + )?; + let validation = sim.validate().await; + assert!( + matches!(validation, Validation::Unverified { .. }), + "fetcher error should yield Unverified: {validation:?}" + ); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn run_into_optimistic_aborts_validation() -> Result<()> { + let token = Address::repeat_byte(0x44); + let owner = Address::repeat_byte(0x55); + let recipient = Address::repeat_byte(0x66); + + let mut cache = cache_with_balance(token, owner, U256::from(1000)).await?; + cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([( + (token, balance_slot_for(owner)), + U256::from(1000), + )]))); + + let mut controller = FreshnessController::new(FreshnessRegistry::new(), AlwaysVerify); + let sim = controller.run( + &mut cache, + vec![SimRequest::new( + owner, + token, + transfer_calldata(recipient, U256::from(100)), + )], + )?; + let results = sim.into_optimistic(); + assert_eq!(results.len(), 1); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn never_verify_skips_predicted_but_reconciles_read_set() -> Result<()> { + // NeverVerify selects nothing from the predicted candidates, but the + // validator still reconciles the actual read set, so a real change is caught. + let token = Address::repeat_byte(0x44); + let owner = Address::repeat_byte(0x55); + let recipient = Address::repeat_byte(0x66); + + // Owner funded so the optimistic transfer succeeds and the balance slot is + // captured in the read set; the fetcher then reports a changed value. + let mut cache = cache_with_balance(token, owner, U256::from(1000)).await?; + cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([( + (token, balance_slot_for(owner)), + U256::from(50), + )]))); + + let mut controller = FreshnessController::new(FreshnessRegistry::new(), NeverVerify); + let sim = controller.run( + &mut cache, + vec![SimRequest::new( + owner, + token, + transfer_calldata(recipient, U256::from(100)), + )], + )?; + let validation = sim.validate().await; + assert!( + matches!(validation, Validation::Corrected { .. }), + "actual-read-set reconcile should still catch the change: {validation:?}" + ); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn pinned_slot_is_not_verified() -> Result<()> { + // Pin the owner's balance slot: even though the fetcher would report a + // change, a pinned slot is excluded from verification → Confirmed. + let token = Address::repeat_byte(0x44); + let owner = Address::repeat_byte(0x55); + let recipient = Address::repeat_byte(0x66); + + let mut cache = cache_with_balance(token, owner, U256::from(1000)).await?; + cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([( + (token, balance_slot_for(owner)), + U256::from(9999), // would be a change if verified + )]))); + + let mut registry = FreshnessRegistry::new(); + registry.pin_slot(token, balance_slot_for(owner)); + let mut controller = FreshnessController::new(registry, AlwaysVerify); + let sim = controller.run( + &mut cache, + vec![SimRequest::new( + owner, + token, + transfer_calldata(recipient, U256::from(100)), + )], + )?; + let validation = sim.validate().await; + assert!( + matches!(validation, Validation::Confirmed), + "pinned slot must not be verified: {validation:?}" + ); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn wall_clock_controller_runs() -> Result<()> { + // Exercise the WallClock variant end-to-end (BlockClock is the default). + let token = Address::repeat_byte(0x44); + let owner = Address::repeat_byte(0x55); + let recipient = Address::repeat_byte(0x66); + + let mut cache = cache_with_balance(token, owner, U256::from(1000)).await?; + cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([( + (token, balance_slot_for(owner)), + U256::from(1000), + )]))); + + let mut controller = + FreshnessController::with_clock(FreshnessRegistry::new(), AlwaysVerify, WallClock); + let sim = controller.run( + &mut cache, + vec![SimRequest::new( + owner, + token, + transfer_calldata(recipient, U256::from(100)), + )], + )?; + let validation = sim.validate().await; + assert!( + matches!(validation, Validation::Confirmed), + "{validation:?}" + ); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn valid_through_becomes_volatile_after_boundary() -> Result<()> { + use evm_fork_cache::freshness::BlockClock; + + let token = Address::repeat_byte(0x44); + let owner = Address::repeat_byte(0x55); + let recipient = Address::repeat_byte(0x66); + + let mut cache = cache_with_balance(token, owner, U256::from(1000)).await?; + cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([( + (token, balance_slot_for(owner)), + U256::from(2000), // a change, if verified + )]))); + + // Valid through block 100. At block 100 it's still pinned; at 101 volatile. + let mut registry = FreshnessRegistry::new(); + registry.valid_through_slot(token, balance_slot_for(owner), 100); + + let clock = BlockClock::at(100); + let mut controller = FreshnessController::with_clock(registry, AlwaysVerify, clock.clone()); + + // At block 100: still valid → not verified → Confirmed. + let sim = controller.run( + &mut cache, + vec![SimRequest::new( + owner, + token, + transfer_calldata(recipient, U256::from(100)), + )], + )?; + assert!(matches!(sim.validate().await, Validation::Confirmed)); + + // Advance past the boundary: now volatile → the change is caught. + clock.set_block(101); + let sim = controller.run( + &mut cache, + vec![SimRequest::new( + owner, + token, + transfer_calldata(recipient, U256::from(100)), + )], + )?; + assert!( + matches!(sim.validate().await, Validation::Corrected { .. }), + "past ValidThrough boundary the slot is volatile and the change is caught" + ); + Ok(()) +} From 320d1486458e9c833e7cd858594e328aa554744a Mon Sep 17 00:00:00 2001 From: Kai Aldag Date: Mon, 15 Jun 2026 00:32:12 +0100 Subject: [PATCH 06/26] Phase 2 (step 5): docs, offline example, README, re-exports, ROADMAP - Re-export the key freshness types from the crate root and add the module to the crate-level doc. - Add the offline `examples/freshness_optimistic.rs` demonstrating a `Corrected` validation via a stub fetcher, and list it in the README example table. - Flip the ROADMAP Phase 2 status to Done and note what landed. The `freshness` module-level doctest (registry + policy, no network) is the runnable doc example. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 1 + docs/ROADMAP.md | 13 ++- examples/freshness_optimistic.rs | 141 +++++++++++++++++++++++++++++++ src/lib.rs | 8 ++ 4 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 examples/freshness_optimistic.rs diff --git a/README.md b/README.md index 8a18b84..f655332 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,7 @@ and inject all state directly: | `transfer_inspector` | Report per-token balance deltas from a simulation. | | `deploy_and_override` | Deploy from creation code and etch it over another address. | | `prefetch_registry` | Record and persist storage touch sets for cross-cycle prefetch. | +| `freshness_optimistic` | Optimistic verify-and-rerun loop: a `Corrected` validation via a stub fetcher. | RPC-gated examples fork real mainnet state. Set `RPC_URL` to an Ethereum RPC endpoint (they print instructions and exit if it is unset): diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 4a834bd..a7f7f9a 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -71,7 +71,7 @@ RPC node Event-driven sync ← WS logs · new block | --- | --- | --- | | **0** | API hygiene + correctness: drop `amms`, fix `set_block` divergence + `block_in_place` panic, commit the tree. | **Done** (`p0-oss-prep`) | | **1** | Engine seam: typed errors, configurable tx/block env, hot-path benches, builder, `protocols` feature. | **Done** (`phase-1-engine-seam`) | -| **2** | Freshness core (Pillar C): `Validity` + `FreshnessRegistry`; `on_new_block` purge; pin immutables. | Planned | +| **2** | Freshness core (Pillar C): `Validity` + `FreshnessRegistry`; observation tracker; policies; optimistic verify-and-rerun loop. | **Done** (`phase-2-freshness`) | | **3** | State-update primitives (Pillar B.1): `StateUpdate` + targeted writers; refold `inject_*`; surface state-diff output. | Planned | | **4** | Event pipeline + adapters (Pillar B.2): `EventDecoder` trait, V3 adapter, WS ingestion loop, reorg handling. | Planned | | **5** | COW snapshots (Pillar A): structural sharing; overlay buffer reuse. | Planned | @@ -287,11 +287,20 @@ against a **stubbed** `StorageBatchFetchFn` returning chosen "current" values (refresh + selective re-run of only affected sims); `purge_account` drops account + storage on both layers; `ValidThrough` boundary; `WallClock` vs `BlockClock`. -### Acceptance +### Acceptance — met `cargo fmt --check`, `clippy --all-targets -- -D warnings` (default + `--lib --no-default-features`), `cargo test`, `RUSTDOCFLAGS=-D warnings cargo doc`. +Landed on `phase-2-freshness`: `src/freshness.rs` (the generic core — `Validity` +/ `FreshnessRegistry`, `FreshnessClock` + `BlockClock`/`WallClock`, +`FreshnessParams`, `FreshnessPolicy` + `AlwaysVerify`/`NeverVerify`/ +`ObservationDriven`, `SlotChange`/`Validation`/`SpeculativeSim`/`SimRequest`, +`FreshnessController`); a clock-agnostic `SlotObservationTracker`; +`EvmCache::verify_slots`/`purge_account`/`set_storage_batch_fetcher`; +`EvmSnapshot::storage_value` + `EvmOverlay::override_slot` validator seams; the +offline `examples/freshness_optimistic.rs`; and `tests/freshness.rs`. + --- ## Key abstractions for later phases (sketches) diff --git a/examples/freshness_optimistic.rs b/examples/freshness_optimistic.rs new file mode 100644 index 0000000..274315d --- /dev/null +++ b/examples/freshness_optimistic.rs @@ -0,0 +1,141 @@ +//! Optimistic execution with deferred validation — a `Corrected` verdict. +//! +//! The freshness controller runs a simulation against a frozen snapshot and +//! returns its result *immediately*, while a background task concurrently +//! re-checks the volatile storage the sim read. If a value the sim depended on +//! has changed, the affected sim is re-run with the fresh value and the verdict +//! is [`Validation::Corrected`]. +//! +//! Here a MockERC20 holder starts with a balance of 1000, so the optimistic +//! `transfer(100)` succeeds. A **stub fetcher** then reports the balance has +//! dropped to 50 — too small to cover the transfer — so the corrected re-run +//! reverts. One slot is pinned (immutable) to show it is never re-verified. +//! +//! Runs fully offline against a mocked provider and a stubbed +//! `StorageBatchFetchFn`; no network access. +//! +//! Run with: +//! +//! ```sh +//! cargo run --example freshness_optimistic +//! ``` + +use std::collections::HashMap; +use std::sync::Arc; + +use alloy_primitives::{Address, Bytes, U256, keccak256}; +use alloy_sol_types::{SolCall, SolValue}; +use anyhow::Result; +use evm_fork_cache::cache::StorageBatchFetchFn; +use evm_fork_cache::freshness::{ + AlwaysVerify, FreshnessController, FreshnessRegistry, SimRequest, Validation, +}; + +#[path = "support/mock.rs"] +mod mock; + +/// Hashed storage slot of `balanceOf[owner]` (mapping at slot 3). +fn balance_slot(owner: Address) -> U256 { + let key = keccak256((owner, U256::from(mock::MOCK_ERC20_BALANCE_SLOT)).abi_encode()); + U256::from_be_bytes(key.0) +} + +#[tokio::main(flavor = "multi_thread")] +async fn main() -> Result<()> { + let mut cache = mock::offline_cache().await?; + + let token = Address::repeat_byte(0x11); + let owner = Address::repeat_byte(0x22); + let recipient = Address::repeat_byte(0x33); + // Address::ZERO is the default block coinbase, touched for gas accounting. + mock::install_default_account(&mut cache, Address::ZERO); + mock::install_default_account(&mut cache, owner); + mock::install_mock_erc20(&mut cache, token); + + // Owner is funded with 1000 tokens — enough for the optimistic transfer. + let owner_slot = balance_slot(owner); + cache.inject_storage_batch(&[(token, owner_slot, U256::from(1000))]); + + // Stub the batch fetcher: report the owner's balance has DROPPED to 50. + // (An unmapped slot reads as zero, matching how a sim reads an unseen slot.) + let fresh: HashMap<(Address, U256), U256> = + HashMap::from([((token, owner_slot), U256::from(50))]); + let fetcher: StorageBatchFetchFn = Arc::new(move |requests: Vec<(Address, U256)>| { + requests + .into_iter() + .map(|(addr, slot)| { + let value = fresh.get(&(addr, slot)).copied().unwrap_or(U256::ZERO); + (addr, slot, Ok(value)) + }) + .collect() + }); + cache.set_storage_batch_fetcher(fetcher); + + // Classification: the balance slot is volatile (default), and slot 6 (a + // would-be immutable like `token0`) is pinned so it is never re-verified. + let mut registry = FreshnessRegistry::new(); + registry.pin_slot(token, U256::from(6)); + + let mut controller = FreshnessController::new(registry, AlwaysVerify); + + // A non-committing `transfer(recipient, 100)` evaluation sim. + let calldata = Bytes::from( + mock::MockERC20::transferCall { + to: recipient, + amount: U256::from(100), + } + .abi_encode(), + ); + let request = SimRequest::new(owner, token, calldata); + + // run() returns as soon as the optimistic sim finishes — without awaiting RPC. + let sim = controller.run(&mut cache, vec![request])?; + + let optimistic = &sim.optimistic()[0]; + let optimistic_succeeded = !optimistic.logs.is_empty(); + println!("optimistic result (computed immediately, against the snapshot):"); + println!(" gas_used = {}", optimistic.gas_used); + println!( + " transfer {} (emitted {} log(s))\n", + if optimistic_succeeded { + "SUCCEEDED" + } else { + "reverted" + }, + optimistic.logs.len() + ); + + // Now await the deferred validation verdict. + match sim.validate().await { + Validation::Confirmed => { + println!("validation: Confirmed — nothing the sim read had changed"); + } + Validation::Corrected { results, changed } => { + println!("validation: Corrected — a slot the sim read had changed:"); + for c in &changed { + println!(" {} slot {} : {} -> {}", c.address, c.slot, c.old, c.new); + } + let corrected = &results[0]; + let corrected_succeeded = !corrected.logs.is_empty(); + println!( + "\ncorrected re-run: gas_used = {}, transfer {} (emitted {} log(s))", + corrected.gas_used, + if corrected_succeeded { + "SUCCEEDED" + } else { + "REVERTED (insufficient fresh balance)" + }, + corrected.logs.len() + ); + assert!( + optimistic_succeeded && !corrected_succeeded, + "this example demonstrates an optimistic success corrected to a revert" + ); + } + Validation::Unverified { reason } => { + println!("validation: Unverified — {reason}"); + } + } + + Ok(()) +} diff --git a/src/lib.rs b/src/lib.rs index 15711cb..1025ff0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,6 +18,9 @@ //! warm-slot tracking for gas estimation. //! - [`errors`] — structured simulation errors and an extensible revert-reason //! decoder you can teach your own custom Solidity error selectors. +//! - [`freshness`] — the four-layer freshness model (classification, observation, +//! policy, mechanism) and the optimistic verify-and-rerun execution loop with +//! deferred validation. //! - [`inspector`] — an `Inspector` that captures ERC20 `Transfer` events to //! reconstruct balance deltas from a simulation. //! - [`multicall`] — batched read-only calls. @@ -40,3 +43,8 @@ pub mod multicall; pub mod prefetch_registry; pub use access_set::StorageAccessList; +pub use freshness::{ + AlwaysVerify, BlockClock, FreshnessClock, FreshnessController, FreshnessParams, + FreshnessPolicy, FreshnessRegistry, NeverVerify, ObservationDriven, SimRequest, SlotChange, + SpeculativeSim, Validation, Validity, WallClock, +}; From dc375f0acf8a44260d32e704cee6852d5d05878b Mon Sep 17 00:00:00 2001 From: Kai Aldag Date: Mon, 15 Jun 2026 00:51:47 +0100 Subject: [PATCH 07/26] Phase 2 review: output field, controller cleanup, clock advance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit C1: add `pub output: Bytes` to CallSimulationResult, populated in all four constructors (two simulate methods in cache/mod.rs, the overlay simulate path, and result_to_sim). Maps Success/Revert payloads, empty on Halt, so a corrected view-call's new return value is observable. C2: remove the inert `params` field + `with_params` from FreshnessController (params belong to ObservationDriven); update phase-2-spec §7. C3: FreshnessClock::advance default no-op; BlockClock::advance sets the block; on_new_block advances the clock then notifies the policy, so ValidThrough aging and reuse-window progress flow through the natural API. C4: poison-tolerant freshness mutex locks (unwrap_or_else into_inner). D1: ROADMAP verify_slots bullet no longer claims it observes the tracker. D2: spec notes the new output field and the corrected on_new_block behavior. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/ROADMAP.md | 5 ++-- docs/phase-2-spec.md | 17 ++++++++++-- src/cache/mod.rs | 29 +++++++++++++++---- src/cache/overlay.rs | 8 +++++- src/freshness.rs | 66 ++++++++++++++++++++++++++++---------------- 5 files changed, 90 insertions(+), 35 deletions(-) diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index a7f7f9a..92d6781 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -226,8 +226,9 @@ pub struct FreshnessController { /* regis ### Primitives (on `EvmCache`) - `verify_slots(&mut self, slots) -> Vec` — re-fetch current values via - the existing batched `StorageBatchFetchFn`, compare to cached values, inject the - changed ones, and `observe` each (updating the tracker). Returns the changed set. + the existing batched `StorageBatchFetchFn`, compare to cached values, and inject the + changed ones. Returns the changed set. (It does **not** update the observation + tracker — only the background validator observes checked slots.) - `purge_account(&mut self, addr)` — remove `addr` from the CacheDB overlay, the BlockchainDb accounts map, and its storage, so the next access re-fetches a clean `AccountInfo`. Distinct from storage-only `purge_pool_storage`. diff --git a/docs/phase-2-spec.md b/docs/phase-2-spec.md index 8d6fb3d..c61f19c 100644 --- a/docs/phase-2-spec.md +++ b/docs/phase-2-spec.md @@ -171,7 +171,11 @@ impl SpeculativeSim { impl Drop for SpeculativeSim { /* abort the background task */ } ``` `CallSimulationResult` must be `Clone` (verify it already is; add derive if needed) -so optimistic + corrected copies can coexist and cross the task boundary. +so optimistic + corrected copies can coexist and cross the task boundary. It also +carries a `pub output: Bytes` field (the call's raw return data: the `Success` +payload, the `Revert` payload, or empty on `Halt`), so a corrected **view-call** +re-run that returns a new value is observable even when both runs succeed — +`Corrected.results[i].output` differs from `optimistic[i].output`. ### 4.6 Request @@ -212,11 +216,14 @@ pub struct FreshnessController { tracker: Arc>, policy: P, clock: C, - params: FreshnessParams, pending: Arc>>, // corrections flowing back from bg tasks } ``` +Adaptive thresholds (`FreshnessParams`) are **not** a controller field — they +live on the policy that consumes them (`ObservationDriven { params }`), so the +controller never carries an unused copy. + `run(&mut self, cache: &mut EvmCache, requests: Vec) -> Result` (main thread): 1. **Drain `pending`** into `cache.inject_storage_batch(...)` (apply corrections from @@ -248,7 +255,11 @@ pub struct FreshnessController { re-run ones replaced). 5. On fetcher error → `Validation::Unverified { reason }` (do not trust silently). -`on_new_block(&mut self, block: u64)`: `clock` advance (if `BlockClock`), `policy.on_new_block(block)`. +`on_new_block(&mut self, block: u64)`: advance the clock via +`FreshnessClock::advance(block)` (a no-op for `WallClock`, a `set_block` for +`BlockClock`), then `policy.on_new_block(block)`. Advancing the clock ages +`ValidThrough` slots into `Volatile` and progresses the reuse window through the +natural API — callers do not bump a `BlockClock` separately. **Concurrency notes:** `tracker` and `pending` are `Arc>` so the background task updates them safely; the live `EvmCache` is never shared across threads. diff --git a/src/cache/mod.rs b/src/cache/mod.rs index 58b3b87..aefb749 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -336,6 +336,13 @@ pub struct CallSimulationResult { /// EIP-2930 access list of all accounts and storage slots touched during simulation. /// Extracted from the EVM journaled state after execution. pub access_list: AccessList, + /// Raw return data of the call. + /// + /// `Success` carries the returned bytes, `Revert` the revert payload, and + /// `Halt` an empty slice. This makes a corrected view-call result observable: + /// when a re-run reads a changed slot, the new return value differs here even + /// if both runs succeed. + pub output: Bytes, } sol!( @@ -2110,8 +2117,13 @@ impl EvmCache { let result = evm .transact_one(tx) .map_err(|e| anyhow!("Failed to transact: {:?}", e))?; - let (logs, gas_used) = match result { - ExecutionResult::Success { logs, gas_used, .. } => (logs, gas_used), + let (logs, gas_used, output) = match result { + ExecutionResult::Success { + logs, + gas_used, + output, + .. + } => (logs, gas_used, output.into_data()), _ => return Err(anyhow!("Failed to call: {:?}", result)), }; @@ -2122,11 +2134,11 @@ impl EvmCache { token_deltas.insert(*token, I256::from_raw(post) - I256::from_raw(pre)); } - Ok((gas_used, token_deltas, logs)) + Ok((gas_used, token_deltas, logs, output)) })(); match result { - Ok((gas_used, token_deltas, logs)) => { + Ok((gas_used, token_deltas, logs, output)) => { if commit { evm.commit_inner(); } else { @@ -2137,6 +2149,7 @@ impl EvmCache { token_deltas, logs, access_list: AccessList::default(), + output, }) } Err(err) => { @@ -2175,7 +2188,12 @@ impl EvmCache { .map_err(|e| SimError::Other(anyhow!("Failed to transact: {:?}", e))); match result { - Ok(ExecutionResult::Success { logs, gas_used, .. }) => { + Ok(ExecutionResult::Success { + logs, + gas_used, + output, + .. + }) => { // Compute balance deltas from captured transfers let token_deltas = if let Some(token_list) = tokens { evm.inspector.balance_deltas_for_tokens(owner, token_list) @@ -2206,6 +2224,7 @@ impl EvmCache { token_deltas, logs, access_list, + output: output.into_data(), }) } Ok(ExecutionResult::Revert { gas_used, output }) => { diff --git a/src/cache/overlay.rs b/src/cache/overlay.rs index 9af9514..bedd993 100644 --- a/src/cache/overlay.rs +++ b/src/cache/overlay.rs @@ -255,7 +255,12 @@ impl EvmOverlay { .map_err(|e| SimError::Other(anyhow!("Failed to transact: {:?}", e))); match result { - Ok(ExecutionResult::Success { logs, gas_used, .. }) => { + Ok(ExecutionResult::Success { + logs, + gas_used, + output, + .. + }) => { let token_deltas = if let Some(token_list) = tokens { evm.inspector.balance_deltas_for_tokens(owner, token_list) } else { @@ -276,6 +281,7 @@ impl EvmOverlay { token_deltas, logs, access_list, + output: output.into_data(), }) } Ok(ExecutionResult::Revert { gas_used, output }) => { diff --git a/src/freshness.rs b/src/freshness.rs index aa74994..7267543 100644 --- a/src/freshness.rs +++ b/src/freshness.rs @@ -279,6 +279,13 @@ impl FreshnessRegistry { pub trait FreshnessClock: Send + Sync { /// The current clock value (block number or unix seconds). fn now(&self) -> u64; + + /// Advance the clock to `now`. + /// + /// Called by [`FreshnessController::on_new_block`] so the natural API drives + /// the clock forward. The default is a no-op (for clocks like [`WallClock`] + /// that advance on their own); [`BlockClock`] overrides it to set the block. + fn advance(&self, _now: u64) {} } /// Block-number clock (the default). Cloning shares the underlying counter, so a @@ -308,6 +315,11 @@ impl FreshnessClock for BlockClock { fn now(&self) -> u64 { self.0.load(Ordering::Relaxed) } + + /// Set the current block to `now` (shared across clones). + fn advance(&self, now: u64) { + self.set_block(now); + } } /// Wall-clock clock: [`now`](FreshnessClock::now) returns unix seconds. @@ -564,10 +576,11 @@ impl Drop for SpeculativeSim { /// Drives the optimistic verify-and-rerun loop over an [`EvmCache`]. /// /// Holds the freshness [`FreshnessRegistry`], the shared -/// [`SlotObservationTracker`], a [`FreshnessPolicy`], a [`FreshnessClock`], the -/// [`FreshnessParams`], and the pending-corrections queue. The tracker and the -/// pending queue are `Arc>` so the background validator can update them -/// without touching the `!Send` cache. +/// [`SlotObservationTracker`], a [`FreshnessPolicy`], a [`FreshnessClock`], and +/// the pending-corrections queue. The tracker and the pending queue are +/// `Arc>` so the background validator can update them without touching +/// the `!Send` cache. Adaptive thresholds ([`FreshnessParams`]) live on the +/// policy that uses them ([`ObservationDriven`]), not on the controller. /// /// # Runtime requirement /// [`run`](Self::run) spawns a background task and the (synchronous) fetcher uses @@ -579,7 +592,6 @@ pub struct FreshnessController { tracker: Arc>, policy: P, clock: C, - params: FreshnessParams, pending: Arc>>, } @@ -598,17 +610,10 @@ impl FreshnessController { tracker: Arc::new(Mutex::new(SlotObservationTracker::new())), policy, clock, - params: FreshnessParams::default(), pending: Arc::new(Mutex::new(Vec::new())), } } - /// Replace the [`FreshnessParams`] used by the observation tracker. - pub fn with_params(mut self, params: FreshnessParams) -> Self { - self.params = params; - self - } - /// Use an existing shared observation tracker (e.g. a persisted one). pub fn with_tracker(mut self, tracker: Arc>) -> Self { self.tracker = tracker; @@ -633,12 +638,17 @@ impl FreshnessController { /// Number of corrections waiting to be drained into the cache on the next /// [`run`](Self::run). pub fn pending_len(&self) -> usize { - self.pending.lock().unwrap().len() + self.pending.lock().unwrap_or_else(|e| e.into_inner()).len() } - /// Advance to a new block: bump a [`BlockClock`] is the caller's job (the - /// clock is shared); this notifies the policy. + /// Advance to a new block. + /// + /// Advances the clock to `block` (a no-op for [`WallClock`], a `set_block` + /// for [`BlockClock`]) and then notifies the policy. Advancing the clock is + /// what ages [`Validity::ValidThrough`] slots into [`Validity::Volatile`] and + /// progresses the observation-tracker reuse window through the natural API. pub fn on_new_block(&mut self, block: u64) { + self.clock.advance(block); self.policy.on_new_block(block); } @@ -661,7 +671,7 @@ impl FreshnessController { // 1. Drain pending corrections into the cache before snapshotting. { - let mut pending = self.pending.lock().unwrap(); + let mut pending = self.pending.lock().unwrap_or_else(|e| e.into_inner()); if !pending.is_empty() { let injects: Vec<(Address, U256, U256)> = pending.iter().map(|c| (c.address, c.slot, c.new)).collect(); @@ -708,7 +718,7 @@ impl FreshnessController { } let candidates: Vec<(Address, U256)> = candidate_set.into_iter().collect(); let verify_set = { - let tracker = self.tracker.lock().unwrap(); + let tracker = self.tracker.lock().unwrap_or_else(|e| e.into_inner()); self.policy.select(&candidates, &tracker, now) }; @@ -807,7 +817,7 @@ fn run_validator(input: ValidatorInput) -> Validation { // Compare against the snapshot, observe each checked slot, collect changes. let mut changed = Vec::new(); { - let mut tracker = tracker.lock().unwrap(); + let mut tracker = tracker.lock().unwrap_or_else(|e| e.into_inner()); for &(addr, slot) in &verify { let new = fresh.get(&(addr, slot)).copied().unwrap_or(U256::ZERO); let old = snapshot.storage_value(addr, slot).unwrap_or(U256::ZERO); @@ -829,7 +839,7 @@ fn run_validator(input: ValidatorInput) -> Validation { // Queue corrections for flow-back into the cache on the next run. { - let mut pending = pending.lock().unwrap(); + let mut pending = pending.lock().unwrap_or_else(|e| e.into_inner()); pending.extend(changed.iter().cloned()); } @@ -862,18 +872,26 @@ fn run_validator(input: ValidatorInput) -> Validation { /// Build a [`CallSimulationResult`] from a non-committing execution result and /// its captured access list. `token_deltas` is empty (the optimistic path does -/// not run transfer tracking); gas and logs come from the execution result. +/// not run transfer tracking); gas, logs, and return data come from the +/// execution result. `output` carries the `Success`/`Revert` payload (empty on +/// `Halt`), so a corrected view-call's new return value is observable here. fn result_to_sim(result: ExecutionResult, access_list: &AccessList) -> CallSimulationResult { - let (gas_used, logs) = match &result { - ExecutionResult::Success { gas_used, logs, .. } => (*gas_used, logs.clone()), - ExecutionResult::Revert { gas_used, .. } => (*gas_used, Vec::new()), - ExecutionResult::Halt { gas_used, .. } => (*gas_used, Vec::new()), + let (gas_used, logs, output) = match result { + ExecutionResult::Success { + gas_used, + logs, + output, + .. + } => (gas_used, logs, output.into_data()), + ExecutionResult::Revert { gas_used, output } => (gas_used, Vec::new(), output), + ExecutionResult::Halt { gas_used, .. } => (gas_used, Vec::new(), Bytes::new()), }; CallSimulationResult { gas_used, token_deltas: HashMap::new(), logs, access_list: access_list.clone(), + output, } } From ed92b1c72e476969293e18e81474c536d555e16e Mon Sep 17 00:00:00 2001 From: Kai Aldag Date: Mon, 15 Jun 2026 01:02:44 +0100 Subject: [PATCH 08/26] Phase 2 review: make the test suite pin freshness contracts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewrites the freshness tests so each would fail on the corresponding regression (verified by mutation), plus two small enabling production changes. Production: - FreshnessController::rerun_count() + an Arc threaded into the validator, incremented once per re-executed sim. Makes selective re-run observable (skip vs identical re-run). - The spawned validator yields once before any work so an abort-on-drop can deterministically cancel it before it touches the tracker or queues a correction (run_validator is otherwise fully synchronous). Tests (tests/freshness.rs unless noted): - T1: a balanceOf VIEW call corrected success→different-success; asserts the output return data differs (new balance) with both runs succeeding — not keyed off logs. - T2: run_mismatch test now asserts rerun_count() == 1, so removing the `intersects` filter (which would re-run both sims) fails the test. - T3: real Drop-abort test (drop with no await; assert pending stays 0 and the fetcher was never reached) + fixed into_optimistic abort test to use a correction-queuing fetcher and assert pending stays 0. - T4: missing-slot-as-zero through the controller → Corrected with old=ZERO. - T5: pending drain alters the SECOND run's RESULT (transfer reverts after drain). - T6: panicking fetcher → Unverified (JoinError); no-fetcher cache → Unverified with the "no storage batch fetcher available" reason. - T7: probabilistic should_refetch unit tests (slot_observations.rs) covering change_rate≈0.15 at now==last_checked, ≈0.01 reuse-then-refetch, and a cycle_interval>1 scaling case; plus an ObservationDriven end-to-end controller test seeding the tracker via with_tracker. - T8: on_new_block ages a ValidThrough slot into volatile via the natural API. - T9: assert optimistic/corrected token_deltas.is_empty() (documented stub). Adds tracking_fetcher/panicking_fetcher helpers to tests/common. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/cache/slot_observations.rs | 85 ++++++ src/freshness.rs | 31 ++- tests/common/mod.rs | 32 +++ tests/freshness.rs | 492 ++++++++++++++++++++++++++++++++- 4 files changed, 630 insertions(+), 10 deletions(-) diff --git a/src/cache/slot_observations.rs b/src/cache/slot_observations.rs index 7f54a2d..c88cc63 100644 --- a/src/cache/slot_observations.rs +++ b/src/cache/slot_observations.rs @@ -420,4 +420,89 @@ mod tests { tracker.observe(a, U256::from(0), U256::from(99), 1); assert_eq!(tracker.last_value(a, U256::from(0)), Some(U256::from(99))); } + + // --- T7: probabilistic should_refetch coverage ------------------------- + + /// Insert a fully-specified observation so the probabilistic branch can be + /// tested with an exact `change_rate = change_count / observation_count` and + /// a known `last_checked`, without replaying an `observe` sequence. + fn seed_obs( + tracker: &mut SlotObservationTracker, + a: Address, + slot: U256, + observation_count: u32, + change_count: u32, + last_checked: u64, + ) { + tracker.observations.insert( + SlotKey { address: a, slot }, + SlotObservation { + last_value: U256::from(1), + observation_count, + change_count, + last_checked, + last_changed: last_checked, + }, + ); + } + + #[test] + fn test_probabilistic_refetches_at_now_equals_last_checked() { + // change_rate = 3/20 = 0.15. At now == last_checked, units_elapsed = 0 so + // cycles_elapsed clamps to 1.0; expected = 0.15 > 0.05 → refetch. + let mut tracker = SlotObservationTracker::new(); + let p = params(); + let a = addr(1); + let slot = U256::from(7); + seed_obs(&mut tracker, a, slot, 20, 3, 100); + // Sanity: this is the probabilistic branch (between never and always). + assert!((3.0_f64 / 20.0) < p.always_refetch_rate); + assert!(tracker.should_refetch(a, slot, 100, &p)); + } + + #[test] + fn test_probabilistic_reuses_then_refetches_after_elapsed() { + // change_rate = 1/100 = 0.01. At now == last_checked, expected = 0.01 < + // 0.05 → reuse. After 10 cycles elapsed (cycle_interval = 1), expected = + // 0.01 * 10 = 0.10 > 0.05 → refetch. Stays within max_reuse (300). + let mut tracker = SlotObservationTracker::new(); + let p = params(); + let a = addr(1); + let slot = U256::from(7); + seed_obs(&mut tracker, a, slot, 100, 1, 100); + + // Immediately: reused. + assert!(!tracker.should_refetch(a, slot, 100, &p)); + // After a few units: still under threshold (0.01 * 4 = 0.04 < 0.05). + assert!(!tracker.should_refetch(a, slot, 104, &p)); + // After enough units: over threshold (0.01 * 10 = 0.10 > 0.05). + assert!(tracker.should_refetch(a, slot, 110, &p)); + } + + #[test] + fn test_probabilistic_cycle_interval_scaling() { + // change_rate = 1/100 = 0.01, cycle_interval = 10. cycles_elapsed = + // units_elapsed / 10, so it takes 10x more elapsed units than a unit + // cycle to cross the 0.05 threshold. + let mut tracker = SlotObservationTracker::new(); + let p = FreshnessParams { + cycle_interval: 10, + ..FreshnessParams::default() + }; + let a = addr(1); + let slot = U256::from(7); + seed_obs(&mut tracker, a, slot, 100, 1, 100); + + // 60 units elapsed → 6 cycles → expected = 0.06 > 0.05 → refetch. + assert!(tracker.should_refetch(a, slot, 160, &p)); + // 40 units elapsed → 4 cycles → expected = 0.04 < 0.05 → reuse. (Under a + // unit cycle_interval this same 40-unit gap would be 40 cycles and would + // refetch — proving the cycle_interval scaling is applied.) + assert!(!tracker.should_refetch(a, slot, 140, &p)); + let unit = FreshnessParams::default(); + assert!( + tracker.should_refetch(a, slot, 140, &unit), + "with cycle_interval = 1 the same elapsed gap refetches" + ); + } } diff --git a/src/freshness.rs b/src/freshness.rs index 7267543..be98df4 100644 --- a/src/freshness.rs +++ b/src/freshness.rs @@ -52,7 +52,7 @@ //! ``` use std::collections::{HashMap, HashSet}; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::time::{SystemTime, UNIX_EPOCH}; @@ -593,6 +593,11 @@ pub struct FreshnessController { policy: P, clock: C, pending: Arc>>, + /// Cumulative count of background re-runs performed by the validator across + /// all `run` calls. Shared with the spawned task; incremented once per + /// re-executed sim. Lets callers observe that selective re-run actually + /// skipped the unaffected sims rather than re-running every one. + rerun_count: Arc, } impl FreshnessController { @@ -611,6 +616,7 @@ impl FreshnessController { policy, clock, pending: Arc::new(Mutex::new(Vec::new())), + rerun_count: Arc::new(AtomicUsize::new(0)), } } @@ -641,6 +647,18 @@ impl FreshnessController { self.pending.lock().unwrap_or_else(|e| e.into_inner()).len() } + /// Cumulative number of background re-runs performed by the validator across + /// all [`run`](Self::run) calls so far. + /// + /// Incremented once per sim that the reconcile step actually re-executes + /// (i.e. whose read set intersected a changed slot). A `Corrected` verdict + /// over `n` requests where only one slot changed advances this by the number + /// of *affected* sims, not by `n` — making the selective-re-run behavior + /// directly observable. + pub fn rerun_count(&self) -> usize { + self.rerun_count.load(Ordering::Relaxed) + } + /// Advance to a new block. /// /// Advances the clock to `block` (a no-op for [`WallClock`], a `set_block` @@ -726,8 +744,15 @@ impl FreshnessController { let registry = self.registry.clone(); let tracker = Arc::clone(&self.tracker); let pending = Arc::clone(&self.pending); + let rerun_count = Arc::clone(&self.rerun_count); let optimistic_for_task = optimistic.clone(); let validation = tokio::spawn(async move { + // Yield once before doing any work. `run_validator` is fully + // synchronous (no `.await` inside), so without an early await point + // an abort-on-drop could not preempt it once polled. Yielding gives + // `into_optimistic`/`Drop` a deterministic chance to cancel the task + // before it touches the tracker or queues a correction. + tokio::task::yield_now().await; run_validator(ValidatorInput { snapshot, fetcher, @@ -736,6 +761,7 @@ impl FreshnessController { registry, tracker, pending, + rerun_count, now, verify_set, optimistic: optimistic_for_task, @@ -758,6 +784,7 @@ struct ValidatorInput { registry: FreshnessRegistry, tracker: Arc>, pending: Arc>>, + rerun_count: Arc, now: u64, verify_set: Vec<(Address, U256)>, optimistic: Vec, @@ -773,6 +800,7 @@ fn run_validator(input: ValidatorInput) -> Validation { registry, tracker, pending, + rerun_count, now, verify_set, optimistic, @@ -856,6 +884,7 @@ fn run_validator(input: ValidatorInput) -> Validation { if !intersects { continue; } + rerun_count.fetch_add(1, Ordering::Relaxed); let mut overlay = EvmOverlay::new(Arc::clone(&snapshot), None); for &(addr, slot, value) in &overrides { overlay.override_slot(addr, slot, value); diff --git a/tests/common/mod.rs b/tests/common/mod.rs index cd638e0..dbdadfb 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -125,6 +125,38 @@ pub fn failing_fetcher() -> StorageBatchFetchFn { }) } +/// Build a stub [`StorageBatchFetchFn`] that reports chosen values *and* flips a +/// shared flag the first time it is called. +/// +/// Used by the Drop-abort test to prove the background validator was cancelled +/// before it ever fetched (so it could not have queued a correction). The +/// returned values otherwise behave exactly like [`stub_fetcher`]. +pub fn tracking_fetcher( + values: HashMap<(Address, U256), U256>, + called: Arc, +) -> StorageBatchFetchFn { + Arc::new(move |requests: Vec<(Address, U256)>| { + called.store(true, std::sync::atomic::Ordering::SeqCst); + requests + .into_iter() + .map(|(addr, slot)| { + let value = values.get(&(addr, slot)).copied().unwrap_or(U256::ZERO); + (addr, slot, Ok(value)) + }) + .collect() + }) +} + +/// Build a stub [`StorageBatchFetchFn`] that panics, to exercise the validator's +/// `JoinError` (`Unverified`) path. +pub fn panicking_fetcher() -> StorageBatchFetchFn { + Arc::new( + |_requests: Vec<(Address, U256)>| -> Vec<(Address, U256, Result)> { + panic!("panicking fetcher: deliberate failure for the Unverified test") + }, + ) +} + /// Submit a `transfer(to, amount)` to a `MockERC20`, committing the state change. pub fn transfer( cache: &mut EvmCache, diff --git a/tests/freshness.rs b/tests/freshness.rs index 030e879..9eb36d1 100644 --- a/tests/freshness.rs +++ b/tests/freshness.rs @@ -8,7 +8,7 @@ mod common; use std::collections::HashMap; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use alloy_primitives::{Address, Bytes, U256}; use alloy_sol_types::SolCall; @@ -16,12 +16,12 @@ use anyhow::Result; use common::{ MOCK_ERC20_BALANCE_SLOT, MockERC20, failing_fetcher, install_default_account, - install_mock_erc20, setup_cache, stub_fetcher, + install_mock_erc20, panicking_fetcher, setup_cache, stub_fetcher, tracking_fetcher, }; -use evm_fork_cache::cache::{EvmCache, EvmOverlay}; +use evm_fork_cache::cache::{EvmCache, EvmOverlay, SlotObservationTracker}; use evm_fork_cache::freshness::{ - AlwaysVerify, FreshnessController, FreshnessRegistry, NeverVerify, SimRequest, Validation, - WallClock, + AlwaysVerify, BlockClock, FreshnessController, FreshnessParams, FreshnessRegistry, NeverVerify, + ObservationDriven, SimRequest, Validation, WallClock, }; /// Hashed storage slot of `balanceOf[owner]` for the MockERC20 fixture. @@ -37,6 +37,27 @@ fn transfer_calldata(to: Address, amount: U256) -> Bytes { Bytes::from(MockERC20::transferCall { to, amount }.abi_encode()) } +/// Encode a `balanceOf(account)` view call. +fn balance_of_calldata(account: Address) -> Bytes { + Bytes::from(MockERC20::balanceOfCall { account }.abi_encode()) +} + +/// Decode a `balanceOf` return value from a [`CallSimulationResult`] `output`. +fn decode_balance(output: &Bytes) -> U256 { + MockERC20::balanceOfCall::abi_decode_returns(output).expect("decode balanceOf return") +} + +/// Yield and briefly sleep so that any background validation task that survived +/// (i.e. was *not* aborted) would get a chance to run and mutate shared state. +/// Used by the abort tests: if the task were alive it would queue a correction +/// within this window, so a subsequent `pending_len() == 0` assertion is +/// meaningful rather than merely racing the spawn. +async fn settle() { + tokio::task::yield_now().await; + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + tokio::task::yield_now().await; +} + // --------------------------------------------------------------------------- // EvmCache::verify_slots // --------------------------------------------------------------------------- @@ -386,6 +407,16 @@ async fn run_mismatch_path_corrected_only_affected_rerun() -> Result<()> { !opt[1].logs.is_empty(), "req2 optimistic should succeed (a log)" ); + // T9: the optimistic path does not run transfer tracking, so token_deltas is + // always empty — pin that documented stub behavior on both results. + assert!( + opt[0].token_deltas.is_empty(), + "optimistic token_deltas are empty (no transfer tracking)" + ); + assert!( + opt[1].token_deltas.is_empty(), + "optimistic token_deltas empty" + ); let validation = sim.validate().await; match validation { @@ -415,6 +446,86 @@ async fn run_mismatch_path_corrected_only_affected_rerun() -> Result<()> { // req2's slot did not change → its result is untouched (== optimistic). assert_eq!(results[1].gas_used, opt[1].gas_used, "req2 not re-run"); assert_eq!(results[1].logs.len(), opt[1].logs.len(), "req2 unchanged"); + + // T9: the corrected re-run also skips transfer tracking → empty deltas. + assert!( + results[0].token_deltas.is_empty(), + "corrected result token_deltas are empty (no transfer tracking)" + ); + } + other => panic!("expected Corrected, got {other:?}"), + } + + // The discriminating assertion: exactly ONE sim (req1) was re-run. If the + // `intersects` filter were removed, the validator would re-run BOTH req1 and + // req2, and this would be 2 — so this test fails on that regression. The + // value-equality checks above alone cannot tell a skip from an identical + // re-run; the counter can. + assert_eq!( + controller.rerun_count(), + 1, + "only the affected sim (req1) should be re-run, not req2" + ); + Ok(()) +} + +// T1: a VIEW call corrected from one success to a *different* success. The +// observable return data (not logs) carries the change. +#[tokio::test(flavor = "multi_thread")] +async fn run_view_call_corrected_success_to_different_success() -> Result<()> { + let token = Address::repeat_byte(0x44); + let owner = Address::repeat_byte(0x55); + + // Cache holds balanceOf(owner) == 1000. + let mut cache = cache_with_balance(token, owner, U256::from(1000)).await?; + // Fetcher reports the balance slot changed to 250 (still a success on re-run, + // but a different return value). + cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([( + (token, balance_slot_for(owner)), + U256::from(250), + )]))); + + let mut controller = FreshnessController::new(FreshnessRegistry::new(), AlwaysVerify); + // A pure view call: balanceOf(owner). Its return value depends on the slot. + let req = SimRequest::new(owner, token, balance_of_calldata(owner)); + let sim = controller.run(&mut cache, vec![req])?; + + // Optimistic view call succeeds and returns the OLD balance (1000). + let opt = sim.optimistic().to_vec(); + assert_eq!(opt.len(), 1); + assert!(!opt[0].output.is_empty(), "view call returns data"); + assert_eq!( + decode_balance(&opt[0].output), + U256::from(1000), + "optimistic returns the old balance" + ); + + let validation = sim.validate().await; + match validation { + Validation::Corrected { results, changed } => { + assert_eq!(changed.len(), 1, "exactly the balance slot changed"); + assert_eq!(changed[0].address, token); + assert_eq!(changed[0].slot, balance_slot_for(owner)); + assert_eq!(changed[0].old, U256::from(1000)); + assert_eq!(changed[0].new, U256::from(250)); + + // The corrected re-run STILL succeeds (a balanceOf view never reverts) + // but its return data reflects the NEW balance. + assert_eq!( + decode_balance(&results[0].output), + U256::from(250), + "corrected re-run returns the new balance" + ); + // Both runs succeed (non-empty return data) yet the outputs differ — + // this is the success→different-success contract, not keyed off logs. + assert!( + !results[0].output.is_empty(), + "corrected run still succeeds" + ); + assert_ne!( + results[0].output, opt[0].output, + "corrected output differs from optimistic output" + ); } other => panic!("expected Corrected, got {other:?}"), } @@ -516,6 +627,9 @@ async fn run_unverified_on_fetcher_error() -> Result<()> { Ok(()) } +// T3 (part 2): into_optimistic aborts the validation task. The fetcher WOULD +// queue a correction (it reports a changed value), so if the abort failed we +// would observe a non-zero pending queue. We assert it stays 0. #[tokio::test(flavor = "multi_thread")] async fn run_into_optimistic_aborts_validation() -> Result<()> { let token = Address::repeat_byte(0x44); @@ -523,9 +637,10 @@ async fn run_into_optimistic_aborts_validation() -> Result<()> { let recipient = Address::repeat_byte(0x66); let mut cache = cache_with_balance(token, owner, U256::from(1000)).await?; + // A CHANGED value: if the validator ran, it would queue a correction. cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([( (token, balance_slot_for(owner)), - U256::from(1000), + U256::from(50), )]))); let mut controller = FreshnessController::new(FreshnessRegistry::new(), AlwaysVerify); @@ -537,8 +652,64 @@ async fn run_into_optimistic_aborts_validation() -> Result<()> { transfer_calldata(recipient, U256::from(100)), )], )?; - let results = sim.into_optimistic(); + let results = sim.into_optimistic(); // aborts the background validation assert_eq!(results.len(), 1); + + // Give any (incorrectly) surviving task a chance to run, then assert no + // correction was queued and no re-run happened. + settle().await; + assert_eq!( + controller.pending_len(), + 0, + "into_optimistic must abort validation before it queues a correction" + ); + assert_eq!(controller.rerun_count(), 0, "no re-run after abort"); + Ok(()) +} + +// T3 (part 1): dropping the SpeculativeSim (no validate/into_optimistic) aborts +// the validation task before it can push a correction. The fetcher reports a +// CHANGED value and flips a "called" flag; after the drop + settle we assert the +// pending queue is empty (and, robustly, that the fetcher was never even +// reached) — proving the abort beat the push. +#[tokio::test(flavor = "multi_thread")] +async fn dropping_speculative_sim_aborts_before_queueing_correction() -> Result<()> { + let token = Address::repeat_byte(0x44); + let owner = Address::repeat_byte(0x55); + let recipient = Address::repeat_byte(0x66); + + let mut cache = cache_with_balance(token, owner, U256::from(1000)).await?; + let called = Arc::new(std::sync::atomic::AtomicBool::new(false)); + cache.set_storage_batch_fetcher(tracking_fetcher( + HashMap::from([((token, balance_slot_for(owner)), U256::from(50))]), + Arc::clone(&called), + )); + + let mut controller = FreshnessController::new(FreshnessRegistry::new(), AlwaysVerify); + let sim = controller.run( + &mut cache, + vec![SimRequest::new( + owner, + token, + transfer_calldata(recipient, U256::from(100)), + )], + )?; + // Drop immediately, with NO intervening await, so the abort flag is set + // before the spawned task is ever polled. + drop(sim); + + settle().await; + + assert_eq!( + controller.pending_len(), + 0, + "dropping the sim must abort validation before it queues a correction" + ); + assert!( + !called.load(std::sync::atomic::Ordering::SeqCst), + "the aborted validator should never have reached the fetcher" + ); + assert_eq!(controller.rerun_count(), 0, "no re-run after abort"); Ok(()) } @@ -641,8 +812,6 @@ async fn wall_clock_controller_runs() -> Result<()> { #[tokio::test(flavor = "multi_thread")] async fn valid_through_becomes_volatile_after_boundary() -> Result<()> { - use evm_fork_cache::freshness::BlockClock; - let token = Address::repeat_byte(0x44); let owner = Address::repeat_byte(0x55); let recipient = Address::repeat_byte(0x66); @@ -687,3 +856,308 @@ async fn valid_through_becomes_volatile_after_boundary() -> Result<()> { ); Ok(()) } + +// T4: a sim reads a slot absent from both the snapshot and the cache; the +// fetcher returns a NONZERO value. The validator must treat the missing slot as +// zero and report a SlotChange { old: ZERO, new: nonzero } through the +// controller. +#[tokio::test(flavor = "multi_thread")] +async fn run_missing_slot_treated_as_zero_is_corrected() -> Result<()> { + let token = Address::repeat_byte(0x44); + let owner = Address::repeat_byte(0x55); + + // Owner's balance slot is NEVER injected → snapshot/cache have no entry, so + // the optimistic balanceOf reads it as zero. + let mut cache = setup_cache().await?; + install_default_account(&mut cache, Address::ZERO); + install_default_account(&mut cache, owner); + install_mock_erc20(&mut cache, token); + + let slot = balance_slot_for(owner); + // Fetcher reports a NONZERO current value for the unseen slot. + cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([( + (token, slot), + U256::from(777), + )]))); + + let mut controller = FreshnessController::new(FreshnessRegistry::new(), AlwaysVerify); + let req = SimRequest::new(owner, token, balance_of_calldata(owner)); + let sim = controller.run(&mut cache, vec![req])?; + + // Optimistic reads the unseen slot as zero. + let opt = sim.optimistic().to_vec(); + assert_eq!( + decode_balance(&opt[0].output), + U256::ZERO, + "unseen slot reads as zero optimistically" + ); + + match sim.validate().await { + Validation::Corrected { results, changed } => { + let change = changed + .iter() + .find(|c| c.address == token && c.slot == slot) + .expect("the missing balance slot should be reported as changed"); + assert_eq!(change.old, U256::ZERO, "missing slot treated as old = zero"); + assert_eq!(change.new, U256::from(777), "fetcher's nonzero value"); + // The corrected re-run now sees the fresh balance. + assert_eq!( + decode_balance(&results[0].output), + U256::from(777), + "corrected re-run returns the fresh balance" + ); + } + other => panic!("expected Corrected, got {other:?}"), + } + Ok(()) +} + +// T5: a queued correction, once drained on the SECOND run, changes the second +// run's *result* (not merely the cached value / verdict). First run queues a +// drop to balance 50; the second run's transfer of 100 then reverts after the +// drain. +#[tokio::test(flavor = "multi_thread")] +async fn pending_drain_alters_subsequent_result() -> Result<()> { + let token = Address::repeat_byte(0x44); + let owner = Address::repeat_byte(0x55); + let recipient = Address::repeat_byte(0x66); + + // Cache holds balance 1000. + let mut cache = cache_with_balance(token, owner, U256::from(1000)).await?; + // Fetcher reports the balance DROPPED to 50 (a change → queued correction). + cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([( + (token, balance_slot_for(owner)), + U256::from(50), + )]))); + + let mut controller = FreshnessController::new(FreshnessRegistry::new(), AlwaysVerify); + + // First run: optimistic transfer of 100 SUCCEEDS against the cached 1000. + let sim = controller.run( + &mut cache, + vec![SimRequest::new( + owner, + token, + transfer_calldata(recipient, U256::from(100)), + )], + )?; + assert!( + !sim.optimistic()[0].logs.is_empty(), + "first-run optimistic transfer succeeds against cached 1000" + ); + assert!(matches!(sim.validate().await, Validation::Corrected { .. })); + assert_eq!(controller.pending_len(), 1, "a correction (→50) is queued"); + + // Second run drains the correction (balance := 50) BEFORE snapshotting, so + // the optimistic transfer of 100 now REVERTS against the drained 50. + let sim = controller.run( + &mut cache, + vec![SimRequest::new( + owner, + token, + transfer_calldata(recipient, U256::from(100)), + )], + )?; + assert_eq!(controller.pending_len(), 0, "pending drained"); + assert!( + sim.optimistic()[0].logs.is_empty(), + "second-run optimistic transfer REVERTS — the drained value (50 < 100) \ + changed the *result*, not just the cached value" + ); + Ok(()) +} + +// T6a: a panicking fetcher → the validator task panics → JoinError → Unverified. +#[tokio::test(flavor = "multi_thread")] +async fn run_unverified_on_fetcher_panic() -> Result<()> { + let token = Address::repeat_byte(0x44); + let owner = Address::repeat_byte(0x55); + let recipient = Address::repeat_byte(0x66); + + let mut cache = cache_with_balance(token, owner, U256::from(1000)).await?; + cache.set_storage_batch_fetcher(panicking_fetcher()); + + let mut controller = FreshnessController::new(FreshnessRegistry::new(), AlwaysVerify); + let sim = controller.run( + &mut cache, + vec![SimRequest::new( + owner, + token, + transfer_calldata(recipient, U256::from(100)), + )], + )?; + let validation = sim.validate().await; + assert!( + matches!(validation, Validation::Unverified { .. }), + "a panicking fetcher (JoinError) should yield Unverified: {validation:?}" + ); + Ok(()) +} + +// T6b: a cache with NO storage batch fetcher → Unverified with the specific +// "no storage batch fetcher available" reason. +#[tokio::test(flavor = "multi_thread")] +async fn run_unverified_without_fetcher() -> Result<()> { + use revm::primitives::hardfork::SpecId; + + let token = Address::repeat_byte(0x44); + let owner = Address::repeat_byte(0x55); + let recipient = Address::repeat_byte(0x66); + + // A `from_backend` cache exposes no fetcher (no provider captured). + let base = cache_with_balance(token, owner, U256::from(1000)).await?; + let mut cache = EvmCache::from_backend( + base.backend().clone(), + base.blockchain_db().clone(), + None, + base.chain_id(), + None, + None, + SpecId::CANCUN, + ); + // Seed the same state the simulation needs into the no-fetcher cache. + install_default_account(&mut cache, Address::ZERO); + install_default_account(&mut cache, owner); + install_mock_erc20(&mut cache, token); + cache.inject_storage_batch(&[(token, balance_slot_for(owner), U256::from(1000))]); + assert!( + cache.storage_batch_fetcher().is_none(), + "from_backend cache has no fetcher" + ); + + let mut controller = FreshnessController::new(FreshnessRegistry::new(), AlwaysVerify); + let sim = controller.run( + &mut cache, + vec![SimRequest::new( + owner, + token, + transfer_calldata(recipient, U256::from(100)), + )], + )?; + match sim.validate().await { + Validation::Unverified { reason } => { + assert_eq!(reason, "no storage batch fetcher available", "{reason}"); + } + other => panic!("expected Unverified, got {other:?}"), + } + Ok(()) +} + +// T7 (controller-level): drive ObservationDriven end-to-end. Seed the tracker so +// the owner's balance slot is a well-observed, never-changed slot; with the +// adaptive policy it is NOT selected for verification this cycle, yet the +// validator's actual-read-set reconcile still catches the real change. This +// exercises the controller → policy → should_refetch path. +#[tokio::test(flavor = "multi_thread")] +async fn observation_driven_controller_end_to_end() -> Result<()> { + let token = Address::repeat_byte(0x44); + let owner = Address::repeat_byte(0x55); + let recipient = Address::repeat_byte(0x66); + + let mut cache = cache_with_balance(token, owner, U256::from(1000)).await?; + cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([( + (token, balance_slot_for(owner)), + U256::from(50), // a real change + )]))); + + // Seed a shared tracker so the balance slot is "stable, well-observed": + // enough never-changed observations that should_refetch() returns false. + let params = FreshnessParams::default(); + let slot = balance_slot_for(owner); + let tracker = { + let mut t = SlotObservationTracker::new(); + for now in 0..params.min_observations { + t.observe(token, slot, U256::from(1000), now as u64); + } + // At a now within the reuse window, a stable slot is not refetched. + assert!(!t.should_refetch(token, slot, params.min_observations as u64, ¶ms)); + Arc::new(Mutex::new(t)) + }; + + // Use a predicted access list so the policy actually receives the slot as a + // candidate (the predicted set drives policy.select). + use alloy_eips::eip2930::{AccessList, AccessListItem}; + let predicted = AccessList(vec![AccessListItem { + address: token, + storage_keys: vec![alloy_primitives::B256::from(slot)], + }]); + + let clock = BlockClock::at(params.min_observations as u64); + let mut controller = FreshnessController::with_clock( + FreshnessRegistry::new(), + ObservationDriven::new(params), + clock, + ) + .with_tracker(Arc::clone(&tracker)); + + let req = SimRequest::new(owner, token, transfer_calldata(recipient, U256::from(100))) + .with_access_list(predicted); + let sim = controller.run(&mut cache, vec![req])?; + + // Even though the policy declined to *predictively* verify the stable slot, + // the validator's actual-read-set reconcile catches the real change. + match sim.validate().await { + Validation::Corrected { changed, .. } => { + assert!( + changed.iter().any(|c| c.address == token && c.slot == slot), + "the actual-read-set reconcile catches the balance change" + ); + } + other => panic!("expected Corrected, got {other:?}"), + } + Ok(()) +} + +// T8: on_new_block advances the BlockClock so a ValidThrough(100) slot becomes +// volatile, driven entirely through the controller's natural API (no separate +// clock bump). +#[tokio::test(flavor = "multi_thread")] +async fn on_new_block_ages_valid_through() -> Result<()> { + let token = Address::repeat_byte(0x44); + let owner = Address::repeat_byte(0x55); + let recipient = Address::repeat_byte(0x66); + + let mut cache = cache_with_balance(token, owner, U256::from(1000)).await?; + cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([( + (token, balance_slot_for(owner)), + U256::from(2000), // a change, if the slot is verified + )]))); + + let mut registry = FreshnessRegistry::new(); + registry.valid_through_slot(token, balance_slot_for(owner), 100); + + // Start at block 100 (still valid). Advance via on_new_block(101) — NOT a + // direct set_block — so the natural API ages the slot into volatile. + let mut controller = + FreshnessController::with_clock(registry, AlwaysVerify, BlockClock::at(100)); + + let sim = controller.run( + &mut cache, + vec![SimRequest::new( + owner, + token, + transfer_calldata(recipient, U256::from(100)), + )], + )?; + assert!( + matches!(sim.validate().await, Validation::Confirmed), + "at block 100 the ValidThrough slot is still pinned" + ); + + // Advance the clock through the controller API. + controller.on_new_block(101); + + let sim = controller.run( + &mut cache, + vec![SimRequest::new( + owner, + token, + transfer_calldata(recipient, U256::from(100)), + )], + )?; + assert!( + matches!(sim.validate().await, Validation::Corrected { .. }), + "after on_new_block(101) the slot is volatile and the change is caught" + ); + Ok(()) +} From 8ab25325dc8b08beb34302cdfc037514aee56462 Mon Sep 17 00:00:00 2001 From: Kai Aldag Date: Mon, 15 Jun 2026 11:53:20 +0100 Subject: [PATCH 09/26] Phase 2: add freshness optimistic-loop benchmarks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Benchmarks the optimistic sims + background slot validation on a swap-shaped sim (MockERC20 transfer reads/writes a balance slot) with correct vs stale snapshots. Two groups, fully offline via stub fetchers: - phase2_cpu: freshness-layer overhead (optimistic run, confirmed/corrected full cycle) with a zero-latency fetcher. - phase2_latency_50ms: latency hiding vs a naive fetch-then-simulate baseline, using a stub fetcher with a 50ms simulated RPC round-trip — optimistic result in ~9us vs ~55ms for the naive path. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.toml | 4 + benches/freshness.rs | 312 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 316 insertions(+) create mode 100644 benches/freshness.rs diff --git a/Cargo.toml b/Cargo.toml index b630671..188f36c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,3 +73,7 @@ harness = false [[bench]] name = "simulation" harness = false + +[[bench]] +name = "freshness" +harness = false diff --git a/benches/freshness.rs b/benches/freshness.rs new file mode 100644 index 0000000..63a08fa --- /dev/null +++ b/benches/freshness.rs @@ -0,0 +1,312 @@ +//! Phase 2 benchmarks: optimistic simulation + background slot validation. +//! +//! The sim is *swap-shaped*: a `MockERC20.transfer` reads the sender's balance +//! slot and writes balances — the same "read a state slot, write new state" +//! shape as a Uniswap pool swap (reads slot0/liquidity, writes new state). The +//! freshness layer treats that read slot as `Volatile` and verifies it. +//! +//! - **Correct snapshot:** the (stub) fetcher reports the read slot unchanged → +//! `Confirmed`, no re-run. +//! - **Stale snapshot:** the fetcher reports the read slot changed → `Corrected`, +//! the affected sim is re-run. +//! +//! Two groups: +//! - `phase2_cpu` (zero-latency stub) — the CPU overhead the freshness layer adds. +//! - `phase2_latency_50ms` (stub with a 50 ms simulated RPC round-trip) — the +//! latency-hiding value prop: time-to-optimistic-result vs time-to-validated vs +//! the naive "fetch-fresh-then-simulate" baseline. +//! +//! Fully offline (mocked provider + stub fetchers), so reproducible. A +//! current-thread runtime is used because the stub fetchers are synchronous; the +//! optimistic loop's deferred validation still works (the validator is a spawned +//! task driven by `validate().await`). + +use std::collections::HashMap; +use std::hint::black_box; +use std::sync::Arc; +use std::time::Duration; + +use alloy_primitives::{Address, Bytes, U256, hex, keccak256}; +use alloy_provider::RootProvider; +use alloy_provider::network::AnyNetwork; +use alloy_rpc_client::RpcClient; +use alloy_sol_types::{SolCall, SolValue, sol}; +use alloy_transport::mock::Asserter; +use criterion::{BatchSize, Criterion, criterion_group, criterion_main}; +use evm_fork_cache::cache::{EvmCache, EvmOverlay, StorageBatchFetchFn}; +use evm_fork_cache::freshness::{ + AlwaysVerify, FreshnessController, FreshnessRegistry, SimRequest, Validation, +}; +use revm::state::{AccountInfo, Bytecode}; +use tokio::runtime::{Builder, Runtime}; + +const MOCK_ERC20_RUNTIME_HEX: &str = include_str!("../fixtures/mock_erc20_runtime.hex"); +const BALANCE_BASE_SLOT: u64 = 3; +const TOKEN: Address = Address::repeat_byte(0xAA); +const SENDER: Address = Address::repeat_byte(0xBB); +const RECIPIENT: Address = Address::repeat_byte(0xCC); + +sol! { + interface MockERC20 { + function transfer(address to, uint256 amount) returns (bool); + } +} + +/// keccak256(abi.encode(owner, 3)) — the `balanceOf(owner)` storage slot. +fn balance_slot(owner: Address) -> U256 { + U256::from_be_bytes(keccak256((owner, U256::from(BALANCE_BASE_SLOT)).abi_encode()).0) +} + +fn current_thread_rt() -> Runtime { + Builder::new_current_thread().enable_all().build().unwrap() +} + +/// A swap-shaped cache: MockERC20 with `SENDER` funded `bal`, `RECIPIENT` zero. +fn swap_cache(rt: &Runtime, bal: u64) -> EvmCache { + let provider = RootProvider::::new(RpcClient::mocked(Asserter::new())); + let mut cache = rt.block_on(EvmCache::new(Arc::new(provider), None)); + let runtime = Bytecode::new_raw(Bytes::from( + hex::decode(MOCK_ERC20_RUNTIME_HEX.trim()).unwrap(), + )); + let code_hash = runtime.hash_slow(); + cache + .db_mut() + .insert_account_info(Address::ZERO, AccountInfo::default()); + cache + .db_mut() + .insert_account_info(SENDER, AccountInfo::default()); + cache + .db_mut() + .insert_account_info(RECIPIENT, AccountInfo::default()); + cache.db_mut().insert_account_info( + TOKEN, + AccountInfo { + balance: U256::ZERO, + nonce: 0, + code: Some(runtime), + code_hash, + account_id: None, + }, + ); + // Mark storage local so unseeded slots read as zero (no RPC fallthrough). + cache + .db_mut() + .replace_account_storage(TOKEN, Default::default()) + .unwrap(); + cache + .insert_mapping_storage_slot( + TOKEN, + U256::from(BALANCE_BASE_SLOT), + SENDER, + U256::from(bal), + ) + .unwrap(); + cache + .insert_mapping_storage_slot(TOKEN, U256::from(BALANCE_BASE_SLOT), RECIPIENT, U256::ZERO) + .unwrap(); + cache +} + +/// A stub fetcher reporting `values` for known slots (zero otherwise), with an +/// optional simulated RPC delay. +fn stub_fetcher( + values: HashMap<(Address, U256), U256>, + delay: Option, +) -> StorageBatchFetchFn { + Arc::new(move |reqs: Vec<(Address, U256)>| { + if let Some(d) = delay { + std::thread::sleep(d); + } + reqs.into_iter() + .map(|(a, s)| (a, s, Ok(values.get(&(a, s)).copied().unwrap_or(U256::ZERO)))) + .collect() + }) +} + +fn transfer_calldata(amount: u64) -> Bytes { + Bytes::from( + MockERC20::transferCall { + to: RECIPIENT, + amount: U256::from(amount), + } + .abi_encode(), + ) +} + +fn controller() -> FreshnessController { + FreshnessController::new(FreshnessRegistry::new(), AlwaysVerify) +} + +/// `reported` is what the fetcher claims the sender balance currently is. +fn fetcher_for(reported: u64, delay: Option) -> StorageBatchFetchFn { + stub_fetcher( + HashMap::from([((TOKEN, balance_slot(SENDER)), U256::from(reported))]), + delay, + ) +} + +fn bench_phase2_cpu(c: &mut Criterion) { + let rt = current_thread_rt(); + let calldata = transfer_calldata(100); + let mut group = c.benchmark_group("phase2_cpu"); + + // Time to the OPTIMISTIC result: snapshot + optimistic sim + read-set capture + // + spawn. The sim is dropped (validator aborted) without awaiting validation. + group.bench_function("optimistic_run", |b| { + b.iter_batched( + || { + let mut cache = swap_cache(&rt, 1000); + cache.set_storage_batch_fetcher(fetcher_for(1000, None)); + (cache, controller()) + }, + |(mut cache, mut ctrl)| { + rt.block_on(async { + let sim = ctrl + .run( + &mut cache, + vec![SimRequest::new(SENDER, TOKEN, calldata.clone())], + ) + .unwrap(); + black_box(sim.optimistic().len()); + }); + }, + BatchSize::SmallInput, + ) + }); + + // Full cycle, CORRECT snapshot → Confirmed (verification matches, no re-run). + group.bench_function("confirmed_correct_snapshot", |b| { + b.iter_batched( + || { + let mut cache = swap_cache(&rt, 1000); + cache.set_storage_batch_fetcher(fetcher_for(1000, None)); + (cache, controller()) + }, + |(mut cache, mut ctrl)| { + rt.block_on(async { + let sim = ctrl + .run( + &mut cache, + vec![SimRequest::new(SENDER, TOKEN, calldata.clone())], + ) + .unwrap(); + black_box(sim.validate().await); + }); + }, + BatchSize::SmallInput, + ) + }); + + // Full cycle, STALE snapshot → Corrected (verification differs, 1 re-run). + group.bench_function("corrected_stale_snapshot", |b| { + b.iter_batched( + || { + let mut cache = swap_cache(&rt, 1000); + cache.set_storage_batch_fetcher(fetcher_for(900, None)); + (cache, controller()) + }, + |(mut cache, mut ctrl)| { + rt.block_on(async { + let sim = ctrl + .run( + &mut cache, + vec![SimRequest::new(SENDER, TOKEN, calldata.clone())], + ) + .unwrap(); + let v = sim.validate().await; + debug_assert!(matches!(v, Validation::Corrected { .. })); + black_box(v); + }); + }, + BatchSize::SmallInput, + ) + }); + + group.finish(); +} + +fn bench_phase2_latency(c: &mut Criterion) { + let rt = current_thread_rt(); + let calldata = transfer_calldata(100); + let delay = Duration::from_millis(50); // simulated RPC round-trip + + let mut group = c.benchmark_group("phase2_latency_50ms"); + group + .sample_size(10) + .warm_up_time(Duration::from_millis(200)) + .measurement_time(Duration::from_secs(3)); + + // NAIVE baseline (the pre-optimistic model): fetch fresh state over RPC, THEN + // simulate. Pays the full RPC latency before any result → ~L + sim. + group.bench_function("naive_fetch_then_sim", |b| { + b.iter_batched( + || { + let mut cache = swap_cache(&rt, 1000); + cache.set_storage_batch_fetcher(fetcher_for(1000, Some(delay))); + cache + }, + |mut cache| { + cache + .verify_slots(&[(TOKEN, balance_slot(SENDER))]) + .unwrap(); // pays L + let snapshot = cache.create_snapshot(); + let mut overlay = EvmOverlay::new(snapshot, None); + black_box(overlay.call_raw(SENDER, TOKEN, calldata.clone()).unwrap()); + }, + BatchSize::SmallInput, + ) + }); + + // OPTIMISTIC: time to the actionable optimistic result. RPC verification has + // not even started (it's a queued task, aborted on drop) → ~sim, NOT L. + group.bench_function("optimistic_time_to_result", |b| { + b.iter_batched( + || { + let mut cache = swap_cache(&rt, 1000); + cache.set_storage_batch_fetcher(fetcher_for(1000, Some(delay))); + (cache, controller()) + }, + |(mut cache, mut ctrl)| { + rt.block_on(async { + let sim = ctrl + .run( + &mut cache, + vec![SimRequest::new(SENDER, TOKEN, calldata.clone())], + ) + .unwrap(); + black_box(sim.optimistic().len()); + }); + }, + BatchSize::SmallInput, + ) + }); + + // OPTIMISTIC, awaiting validation: ~L (the RPC the consumer overlapped with + // its own work). The win is that the result was usable ~L earlier (above). + group.bench_function("optimistic_time_to_validated", |b| { + b.iter_batched( + || { + let mut cache = swap_cache(&rt, 1000); + cache.set_storage_batch_fetcher(fetcher_for(1000, Some(delay))); + (cache, controller()) + }, + |(mut cache, mut ctrl)| { + rt.block_on(async { + let sim = ctrl + .run( + &mut cache, + vec![SimRequest::new(SENDER, TOKEN, calldata.clone())], + ) + .unwrap(); + black_box(sim.validate().await); + }); + }, + BatchSize::SmallInput, + ) + }); + + group.finish(); +} + +criterion_group!(benches, bench_phase2_cpu, bench_phase2_latency); +criterion_main!(benches); From 00653d2b97283c18eed45a9fc6b225ff7792ddbf Mon Sep 17 00:00:00 2001 From: Kai Aldag Date: Mon, 15 Jun 2026 16:19:18 +0100 Subject: [PATCH 10/26] Phase 2 review fixes (F1-F7) + in-progress engine/OSS-prep work Snapshot of the current working tree. Headline change: address the seven Phase 2 review findings, each with a reproduction test added to the freshness suite (25 -> 33 tests): - F1: heal corrections through the CacheDB overlay (inject_storage_batch_fresh) so a validated correction is not shadowed by a stale overlay slot; route the controller drain and verify_slots through it. - F2: iterate the background validator to a fixed point so a correction that flips control flow onto a new volatile slot verifies that slot too (bounded by MAX_VALIDATION_ROUNDS); rerun_count still counts distinct affected sims once. - F3: thread SimRequest.tx (value / gas limit / gas price / nonce / access list) through a new EvmOverlay::call_raw_with_access_list_with; add SimRequest::with_value / with_gas_limit / with_gas_price. - F4: cooperative, best-effort cancellation for the validator (a cancel flag is checked before fetching, before observing, and before queuing corrections) and honest abort rustdoc. - F5: block-aware StorageBatchFetchFn (now takes Option); the controller captures the cache's pinned block at run() so the deferred validator fetches at the snapshot's block, immune to a concurrent set_block re-pin. - F6: SimStatus on CallSimulationResult (Success / Revert / Halt { reason }); CallSimulationResult is now #[non_exhaustive]. The optimistic example branches on status instead of inferring success from logs. - F7: gate protocol-only tests behind the `protocols` feature (file-level on tests/storage_keys.rs; protocol-free unit tests relocated to a core_tests module) so `cargo test --no-default-features` builds and runs. Green: cargo test (default and --no-default-features), clippy --all-targets --all-features -D warnings, fmt --check, RUSTDOCFLAGS=-Dwarnings cargo doc, cargo check --examples (both feature sets), cargo bench --no-run. This commit also captures other in-progress working-tree changes (README, additional examples/tests, and assorted engine/OSS-prep edits) committed in the same snapshot at the author's request. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 68 +++ CONTRIBUTING.md | 99 ++++ Cargo.toml | 14 + README.md | 279 ++++++++---- benches/freshness.rs | 91 +++- benches/rpc_mainnet.rs | 119 +++++ benches/simulation.rs | 153 ++++++- docs/KNOWN_ISSUES.md | 130 ++++++ examples/foundry_artifact_etching.rs | 101 +++++ examples/freshness_multi_sim.rs | 166 +++++++ examples/freshness_optimistic.rs | 27 +- examples/multi_hop_swap.rs | 88 ++++ examples/multicall_with_error_handling.rs | 110 +++++ fixtures/MockERC20.foundry.json | 26 ++ fixtures/README.md | 4 + src/access_list.rs | 100 ++++- src/access_set.rs | 41 +- src/cache/binary_state.rs | 23 +- src/cache/bytecode.rs | 110 +++++ src/cache/metadata.rs | 54 +++ src/cache/mod.rs | 520 ++++++++++++++++++---- src/cache/overlay.rs | 206 ++++++++- src/cache/slot_observations.rs | 38 ++ src/cache/snapshot.rs | 26 +- src/cache/storage_keys.rs | 67 ++- src/cache/tick_snapshot.rs | 38 ++ src/create3.rs | 83 +++- src/deploy.rs | 134 +++++- src/errors.rs | 150 ++++++- src/freshness.rs | 367 ++++++++++++--- src/inspector.rs | 82 +++- src/lib.rs | 109 ++++- src/multicall.rs | 103 ++++- src/prefetch_registry.rs | 74 ++- tests/common/mod.rs | 49 +- tests/errors.rs | 135 ++++++ tests/freshness.rs | 505 ++++++++++++++++++++- tests/multicall.rs | 97 ++++ tests/serialization_roundtrip.rs | 225 ++++++++++ tests/snapshot_overlay.rs | 171 +++++++ tests/storage_keys.rs | 5 + tests/transfer_inspector.rs | 194 ++++++++ 42 files changed, 4793 insertions(+), 388 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 CONTRIBUTING.md create mode 100644 benches/rpc_mainnet.rs create mode 100644 docs/KNOWN_ISSUES.md create mode 100644 examples/foundry_artifact_etching.rs create mode 100644 examples/freshness_multi_sim.rs create mode 100644 examples/multi_hop_swap.rs create mode 100644 examples/multicall_with_error_handling.rs create mode 100644 fixtures/MockERC20.foundry.json create mode 100644 tests/errors.rs create mode 100644 tests/multicall.rs create mode 100644 tests/serialization_roundtrip.rs create mode 100644 tests/snapshot_overlay.rs create mode 100644 tests/transfer_inspector.rs diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..c3229e8 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,68 @@ +# Changelog + +All notable changes to `evm-fork-cache` are documented here. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +**Pre-1.0 policy:** until `1.0.0`, breaking changes may land in **minor** +versions (`0.x.0`); patch versions (`0.x.y`) are non-breaking. The roadmap in +[`docs/ROADMAP.md`](docs/ROADMAP.md) deliberately reshapes the API before the +surface freezes at 1.0. + +## [Unreleased] + +This is the first release line. It captures the work done across the +pre-release development phases (see [`docs/ROADMAP.md`](docs/ROADMAP.md)). + +### Added + +- **Forked EVM cache** (`cache::EvmCache`) backed by `foundry-fork-db` with lazy + RPC loading and on-disk persistence for accounts, storage, bytecode, immutable + metadata, and Uniswap V3-style tick snapshots. +- **`EvmCacheBuilder`** — a fluent constructor (`EvmCache::builder(provider)`) + subsuming the positional `with_cache` / `from_backend` constructors, with + per-instance cache-speed configuration. +- **Snapshots and overlays** — `create_snapshot()` produces an immutable, + `Send + Sync` `EvmSnapshot`; `EvmOverlay` is a cheap per-simulation clone for + isolated parallel evaluation. +- **Freshness control plane** (`freshness` module, Phase 2) — the four-layer + model (`Validity`/`FreshnessRegistry`, `SlotObservationTracker`, + `FreshnessPolicy`, `FreshnessController`), a configurable `FreshnessClock` + (`BlockClock`/`WallClock`), and the optimistic verify-and-rerun execution loop + with deferred validation (`SpeculativeSim`/`Validation`). +- **Freshness primitives on `EvmCache`** — `verify_slots`, `purge_account`, + `set_storage_batch_fetcher`; `EvmOverlay::call_raw_with_access_list` and + `override_slot` for read-set capture and corrected re-runs. +- **Configurable transaction & block environment** — `TxConfig` (value, gas + limit, gas price, nonce, access list) threaded through `call_raw_with`; block + context setters (`set_coinbase`, `set_prevrandao`, `set_block_gas_limit`). +- **Transfer-inspector simulation** (`inspector`) reporting per-token balance + deltas from the `Transfer` event stream. +- **Access-list tooling** (`access_list`, `access_set`) — `StorageAccessList` + touch-set capture, EIP-2930 list construction, and L2 profitability estimation. +- **Multicall3 batching** (`multicall`). +- **Deployment & etching** (`deploy`) — deploy from creation code, etch Foundry + artifacts over forked contracts; **CREATE3** address derivation (`create3`). +- **Extensible revert decoder** (`errors`) — native `Error(string)` / `Panic(uint256)` + decoding plus one-line custom-error registration; typed `SimError` + (`Revert` / `Halt` / `Host`). +- **Two-stage prefetch registry** (`prefetch_registry`) for cross-cycle + storage-slot pre-warming. +- **`protocols` feature** (default-on) gating the Uniswap V2/V3 storage layouts, + V3 tick snapshots, and `inject_v3_*` / `inject_v2_pool_metadata` helpers, so + the generic engine builds with `--no-default-features`. + +### Changed + +- Simulation entry points that distinguish failure modes return + `SimulationResult` (`Result`), separating decoded reverts, + EVM halts, and host errors. `SimulationErrorKind` remains as a deprecated alias. + +### Notes + +- MSRV is Rust 1.88; edition 2024. Both are enforced in CI. +- `EvmCache` requires a multi-thread tokio runtime for any RPC-touching path. +- See [`docs/KNOWN_ISSUES.md`](docs/KNOWN_ISSUES.md) for current limitations. + +[Unreleased]: https://github.com/KaiCode2/evm-fork-cache/commits/main diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..926a005 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,99 @@ +# Contributing to evm-fork-cache + +Thanks for your interest in contributing! This crate is pre-1.0 and developed +against a phased [roadmap](docs/ROADMAP.md). Contributions — bug reports, tests, +docs, examples, and code — are welcome. + +## Getting started + +```sh +git clone https://github.com/KaiCode2/evm-fork-cache +cd evm-fork-cache +cargo test +``` + +The crate is a standalone workspace (it has its own `Cargo.lock`) and needs no +network for the default test suite: every integration test builds the cache over +a mocked provider. A handful of examples and benchmarks fork live mainnet state +behind an `RPC_URL` environment variable and are skipped when it is unset. + +## The green bar + +CI runs the checks below, and every commit on a feature branch is expected to +pass **all** of them. Run them locally before pushing: + +```sh +cargo fmt --all --check +cargo clippy --all-targets --no-deps -- -D warnings +# The generic engine must also build and lint cleanly without the protocols feature: +cargo clippy --lib --no-default-features --no-deps -- -D warnings +cargo test +RUSTDOCFLAGS="-D warnings" cargo doc --no-deps +``` + +A convenience one-liner: + +```sh +cargo fmt --all --check && \ +cargo clippy --all-targets --no-deps -- -D warnings && \ +cargo clippy --lib --no-default-features --no-deps -- -D warnings && \ +cargo test && \ +RUSTDOCFLAGS="-D warnings" cargo doc --no-deps +``` + +### MSRV + +The minimum supported Rust version is **1.88** (edition 2024), enforced by a +dedicated CI job (`cargo check --lib --locked` on 1.88). Do not use std APIs +newer than 1.88 in the library. Dev-only code (examples, benches, tests) is not +MSRV-constrained. + +### Feature configurations + +The `protocols` feature (default on) gates DeFi protocol knowledge. The generic +simulation engine must compile and lint with `--no-default-features`. Any new +DeFi-specific surface (protocol storage layouts, pool injection) must be gated +behind `protocols`; generic machinery stays always-on. When you add a public +item behind `#[cfg(feature = "protocols")]`, also add +`#[cfg_attr(docsrs, doc(cfg(feature = "protocols")))]` so docs.rs renders the +feature badge. + +## Tests, benchmarks, and examples + +- **Tests** live in `tests/` (integration) and inline `#[cfg(test)]` modules + (unit). Shared offline helpers are in `tests/common/`. Keep tests deterministic + and network-free; use the stub `StorageBatchFetchFn` helpers for the freshness + paths. A test should pin a behavior, not merely exercise a code path. +- **Benchmarks** use Criterion and live in `benches/`. Offline benches must stay + reproducible; RPC-gated benches must `return` early (skip, not fail) when + `RPC_URL` is unset, so `cargo bench` is offline by default. +- **Examples** live in `examples/`. Offline examples share `examples/support/mock.rs`. + Each example should explain *what* it shows and *why* it matters, and be listed + in the README table with its network requirement and level. + +## Documentation + +- Document every public item. There is no `missing_docs` gate, but + `cargo doc` runs with `-D warnings`, so broken intra-doc links and malformed + doc comments fail CI. +- Functions returning `Result` should carry an `# Errors` section; functions that + can panic should carry a `# Panics` section. +- Prefer runnable doctests; mark network-dependent snippets `no_run` or `ignore`. + +## Commits and branches + +- Branch from `main` (or the active phase branch). Feature/phase branches follow + the `phase-N-` convention. +- Write focused commits with a clear subject line and a body explaining the *why*. +- Update `CHANGELOG.md` under `[Unreleased]` for any user-visible change. + +## Reporting issues + +Please include the crate version, Rust version, feature flags, and a minimal +reproduction. Known limitations are tracked in +[`docs/KNOWN_ISSUES.md`](docs/KNOWN_ISSUES.md) — check there first. + +## License + +By contributing, you agree that your contributions will be dual-licensed under +the MIT and Apache-2.0 licenses, as described in the [README](README.md#license). diff --git a/Cargo.toml b/Cargo.toml index 188f36c..6ccbb62 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,14 @@ readme = "README.md" repository = "https://github.com/KaiCode2/evm-fork-cache" documentation = "https://docs.rs/evm-fork-cache" +# Build docs.rs with every feature enabled so the `protocols` surface is +# documented, and pass `--cfg docsrs` so feature-gated items render an +# "available on crate feature X" badge (see `#![cfg_attr(docsrs, feature(doc_cfg))]` +# in lib.rs). `docsrs` is only set on docs.rs and never affects local/CI builds. +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] + # Standalone workspace root: keeps this crate from being absorbed by any # ancestor-directory workspace and gives it its own Cargo.lock. [workspace] @@ -77,3 +85,9 @@ harness = false [[bench]] name = "freshness" harness = false + +# RPC-gated real-contract benchmarks. Skipped (not failed) when RPC_URL is unset, +# so `cargo bench` stays offline by default. +[[bench]] +name = "rpc_mainnet" +harness = false diff --git a/README.md b/README.md index f655332..3921531 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,76 @@ # evm-fork-cache -`evm-fork-cache` is a Rust EVM simulation support crate built around `revm`, -`alloy`, and `foundry-fork-db`. It is intended for DeFi search systems that -need repeatable forked-state simulation, low-latency cache reuse, and safe -parallel evaluation of candidate transactions. - -## What It Provides - -- Forked EVM cache backed by `foundry-fork-db` with lazy RPC loading. -- Binary state persistence for accounts, storage, bytecode, immutable metadata, - and Uniswap V3-style tick snapshots. -- Snapshot and overlay APIs for parallel simulations without sharing mutable - REVM state across tasks. -- Direct storage injection and purge helpers for pool-state refresh workflows. -- ERC20 helpers for balances, allowances, decimals, and controlled balance - mutation in simulations. -- Transfer-inspector simulation that reports token balance deltas without - extra pre/post balance queries. -- Storage touch-set capture via `StorageAccessList` for EIP-2929 warm-access - accounting and batch prefetch. -- Multicall3 batching helpers for running many view calls inside the fork. -- Foundry artifact deployment and etching helpers for installing locally - compiled runtime bytecode into a forked simulator. -- CREATE3 address derivation utilities. -- An extensible revert decoder: the two Solidity built-ins (`Error(string)` and - `Panic(uint256)`) are decoded natively, and you register your own - contract-defined custom errors in one line. - -## Example +[![CI](https://github.com/KaiCode2/evm-fork-cache/actions/workflows/ci.yml/badge.svg)](https://github.com/KaiCode2/evm-fork-cache/actions/workflows/ci.yml) +[![crates.io](https://img.shields.io/crates/v/evm-fork-cache.svg)](https://crates.io/crates/evm-fork-cache) +[![docs.rs](https://img.shields.io/docsrs/evm-fork-cache)](https://docs.rs/evm-fork-cache) +[![License: MIT OR Apache-2.0](https://img.shields.io/badge/license-MIT%20OR%20Apache--2.0-blue.svg)](#license) + +A forked-EVM **simulation engine** for DeFi search, MEV, and backtesting — built +on [`revm`], [`alloy`], and [`foundry-fork-db`]. + +It exists to answer one question fast and repeatedly: *"if I sent this +transaction against current on-chain state, what would happen?"* — for thousands +of candidate transactions per block, without paying an RPC round-trip or +re-deriving state on every call. + +[`revm`]: https://github.com/bluealloy/revm +[`alloy`]: https://github.com/alloy-rs/alloy +[`foundry-fork-db`]: https://github.com/foundry-rs/foundry-fork-db + +## Why it exists + +A DeFi search loop evaluates many hypothetical transactions against the *same* +recent chain state. Doing that with a naive fork means re-fetching state, paying +RPC latency on the hot path, and either sharing mutable EVM state across tasks +(unsafe) or deep-cloning a fork per candidate (slow). `evm-fork-cache` is built +around three capabilities that target exactly this workload: + +1. **Cheap parallel fan-out** — freeze state once into an immutable snapshot, + hand a cheap `Arc` clone to each task, and run many isolated simulations in + parallel. No task can observe another's writes. +2. **Targeted state sync** — refresh or purge *specific* accounts and storage + slots in place (no RPC on the hot path), so hot pool state stays correct + without re-forking. +3. **Freshness as a first-class concept** — the engine tracks what it can trust, + for how long, and verifies the rest. The optimistic verify-and-rerun loop + hides RPC latency: act on speculative results immediately, get a `Confirmed` + or `Corrected` verdict when the background validation lands. + +> **Maturity.** This crate is **pre-1.0** and under active development against a +> [phased roadmap](docs/ROADMAP.md). Capabilities (1) and (3) above are +> implemented today; (2) is partially implemented (targeted purge/inject) with +> the event-driven pipeline still on the roadmap. The public API still changes +> between minor versions — see [Stability](#stability). + +## What it provides today + +- **Forked EVM cache** backed by `foundry-fork-db` with lazy RPC loading and + on-disk persistence for accounts, storage, bytecode, immutable metadata, and + Uniswap V3-style tick snapshots. +- **Snapshots and overlays** — `create_snapshot()` produces an immutable, + `Send + Sync` point-in-time view; each `EvmOverlay` is a cheap clone that + simulates in isolation, ideal for parallel candidate evaluation. +- **Freshness control plane** — a four-layer model (classification, observation, + policy, mechanism) plus an optimistic verify-and-rerun execution loop with + deferred validation. See the [`freshness`](src/freshness.rs) module. +- **Targeted state manipulation** — direct storage injection, account/slot + purge, and balance overrides for pool-state refresh workflows. +- **ERC20 helpers** — balances, allowances, decimals, and controlled balance + mutation (including automatic balance-slot discovery) for simulations. +- **Transfer-inspector simulation** that reports per-token balance deltas + straight from the `Transfer` event stream, no extra pre/post balance queries. +- **Access-list tooling** — `StorageAccessList` captures the EIP-2929 warm-access + touch set; helpers build an EIP-2930 access list and estimate whether attaching + one is profitable on an L2. +- **Multicall3 batching** for running many view calls inside the fork in one pass. +- **Deployment & etching** — deploy from creation code, or etch locally compiled + Foundry runtime bytecode over a forked contract while preserving its storage. +- **CREATE3 address derivation** utilities. +- **An extensible revert decoder** — the two Solidity built-ins (`Error(string)` + and `Panic(uint256)`) decode natively; register your own contract-defined + custom errors in one line. + +## Quick start ```rust,no_run use std::sync::Arc; @@ -43,18 +86,19 @@ let provider = ProviderBuilder::new() .network::() .connect_http("https://example-rpc.invalid".parse()?); -let mut cache = EvmCache::with_cache( - Arc::new(provider), - Some(BlockId::latest()), - None, - SpecId::CANCUN, -) -.await; +// Build a cache pinned to the latest block. (Requires a multi-thread tokio +// runtime — see the note below.) +let mut cache = EvmCache::builder(Arc::new(provider)) + .latest_block() + .spec(SpecId::CANCUN) + .build() + .await; let from = Address::ZERO; let to = Address::repeat_byte(0x11); let calldata = Bytes::new(); +// Simulate, capturing the EIP-2929 touch set as we go. let (_result, touched) = cache.call_raw_with_access_list(from, to, calldata)?; println!( "touched {} accounts and {} storage slots", @@ -65,11 +109,88 @@ println!( # } ``` -## Foundry Artifact Etching +> **Runtime requirement.** `EvmCache` lazily fetches missing state through a +> synchronous façade over an async provider (`tokio::task::block_in_place`), so +> its constructors and any method that may touch RPC must run on a **multi-thread** +> tokio runtime (`#[tokio::main(flavor = "multi_thread")]` or +> `#[tokio::test(flavor = "multi_thread")]`). The offline examples and tests build +> the cache over a mocked provider and never touch the network. + +## Core concepts + +The state stack flows bottom-to-top; reads flow up and the fork DB lazily fetches +misses from RPC: + +``` +EvmOverlay × N isolated, Send simulations (cheap Arc clones) + ▲ clone × N +EvmSnapshot immutable, point-in-time, Send + Sync + ▲ create_snapshot() +EvmCache lazy RPC fetch + local state cache + targeted writes/purge + ▲ lazy fetch +RPC provider +``` + +- **`EvmCache`** owns the mutable fork: it fetches, caches, persists, and applies + targeted writes/purges. It is `!Send` (it block_on's RPC internally). +- **`EvmSnapshot`** is an immutable flattening of the cache at a point in time, + shareable across threads via `Arc`. +- **`EvmOverlay`** wraps a snapshot with a per-simulation dirty layer; clone one + per candidate transaction and simulate without RPC and without touching the + live cache. + +The [`freshness`](src/freshness.rs) module layers a freshness controller on top: +classify each address/slot (`Pinned` / `Volatile` / `ValidThrough`), observe how +often slots change, pick what to verify each cycle with a `FreshnessPolicy`, and +run the optimistic loop that returns speculative results immediately and a +`Confirmed`/`Corrected`/`Unverified` verdict asynchronously. + +## Examples + +The [`examples/`](examples) directory has runnable, documented examples. Run any +with `cargo run --example `. + +**Offline examples** need no network — they build the cache over a mocked provider +and inject all state directly: + +| Example | Level | Shows | +| --- | --- | --- | +| `revert_decoding` | Basic | Decode the standard Solidity `Error`/`Panic`/unknown reverts. | +| `custom_revert_errors` | Basic | Register your own custom Solidity error selectors with `RevertDecoder`. | +| `create3_addresses` | Basic | Derive CREATE3 deployment addresses off-chain. | +| `storage_access_list` | Basic | Merge touch sets, estimate EIP-2929 savings, build an EIP-2930 list. | +| `erc20_balance_override` | Basic | Set an ERC20 balance by scanning for its storage slot. | +| `snapshot_and_restore` | Intermediate | In-place `snapshot()`/`restore()` rollback on one cache. | +| `parallel_overlays` | Intermediate | Fan one `create_snapshot()` out to many isolated `EvmOverlay` simulations. | +| `transfer_inspector` | Intermediate | Report per-token balance deltas from a simulation. | +| `deploy_and_override` | Intermediate | Deploy from creation code and etch it over another address. | +| `foundry_artifact_etching` | Intermediate | Etch a locally compiled Foundry artifact (from a JSON file) over a fork. | +| `prefetch_registry` | Advanced | Record and persist storage touch sets for cross-cycle prefetch. | +| `freshness_optimistic` | Advanced | Optimistic verify-and-rerun loop: a `Corrected` validation via a stub fetcher. | +| `freshness_multi_sim` | Advanced | Many sims with selective re-run, plus classification and `ValidThrough` aging. | + +**RPC examples** fork real mainnet state. Set `RPC_URL` to an Ethereum RPC +endpoint (they print instructions and exit if it is unset): + +| Example | Level | Shows | +| --- | --- | --- | +| `fork_token_balance` | Basic | Lazy RPC loading and warm-cache reuse (cold vs. warm read). | +| `multicall_batch` | Intermediate | Batch many view calls through Multicall3 in one pass. | +| `multicall_with_error_handling` | Intermediate | Batch with `allowFailure`; read partial results when a call reverts. | +| `fork_override_balance` | Intermediate | Discover a real token's balance slot and override it. | +| `multi_hop_swap` | Advanced | Quote a 2-hop Uniswap V2 swap (WETH→USDC→DAI) against live reserves. | + +```sh +cargo run --example revert_decoding +RPC_URL=https://eth.llamarpc.com cargo run --example fork_token_balance +``` + +## Foundry artifact etching Use `etch_foundry_artifact` when replacing an existing forked contract while preserving its storage, balance, and nonce. Use -`etch_foundry_artifact_or_create` for synthetic simulation addresses. +`etch_foundry_artifact_or_create` for synthetic simulation addresses. See the +runnable [`foundry_artifact_etching`](examples/foundry_artifact_etching.rs) example. ```rust,ignore use alloy_primitives::Address; @@ -92,58 +213,64 @@ println!("installed {} bytes at {}", etched.code_size, etched.target_address); # } ``` -## Examples - -The [`examples/`](examples) directory has runnable, documented examples. Run any -with `cargo run --example `. - -Offline examples need no network — they build the cache over a mocked provider -and inject all state directly: +## Benchmarks -| Example | Shows | -| --- | --- | -| `revert_decoding` | Decode the standard Solidity `Error`/`Panic`/unknown reverts. | -| `custom_revert_errors` | Register your own custom Solidity error selectors with `RevertDecoder`. | -| `create3_addresses` | Derive CREATE3 deployment addresses off-chain. | -| `storage_access_list` | Merge touch sets, estimate EIP-2929 savings, build an EIP-2930 list. | -| `erc20_balance_override` | Set an ERC20 balance by scanning for its storage slot. | -| `snapshot_and_restore` | In-place `snapshot()`/`restore()` rollback on one cache. | -| `parallel_overlays` | Fan one `create_snapshot()` out to many isolated `EvmOverlay` simulations. | -| `transfer_inspector` | Report per-token balance deltas from a simulation. | -| `deploy_and_override` | Deploy from creation code and etch it over another address. | -| `prefetch_registry` | Record and persist storage touch sets for cross-cycle prefetch. | -| `freshness_optimistic` | Optimistic verify-and-rerun loop: a `Corrected` validation via a stub fetcher. | - -RPC-gated examples fork real mainnet state. Set `RPC_URL` to an Ethereum RPC -endpoint (they print instructions and exit if it is unset): +Criterion benchmarks live in [`benches/`](benches). The offline benches are the +baseline against which the planned copy-on-write snapshot rewrite (roadmap +Pillar A) will be measured, so they exercise the real hot paths at a range of +cache sizes: -| Example | Shows | +| Bench | Measures | | --- | --- | -| `fork_token_balance` | Lazy RPC loading and warm-cache reuse (cold vs. warm read). | -| `multicall_batch` | Batch many view calls through Multicall3 in one pass. | -| `fork_override_balance` | Discover a real token's balance slot and override it. | +| `simulation` | `create_snapshot` across cache sizes (100 → 10k accounts), overlay fan-out, `call_raw` throughput, sequential bundle execution, batched storage injection. | +| `freshness` | The optimistic loop end-to-end (CPU and latency-hiding), `verify_slots` at scale (1 → 1000 slots), and multi-sim fan-out. | +| `access_list` | Touch-set merge and EIP-2930 list construction. | +| `revert_decoding` | Built-in and custom revert decoding, including decoder dispatch with many registered errors. | +| `storage_keys` | Mapping/array storage-key derivation. | +| `create3` | CREATE3 address derivation. | ```sh -cargo run --example revert_decoding -RPC_URL=https://eth.llamarpc.com cargo run --example fork_token_balance +cargo bench # all offline benches +cargo bench --bench simulation # one suite ``` -## Benchmarks - -Offline Criterion microbenchmarks live in [`benches/`](benches) (revert -decoding, storage-key derivation, CREATE3, and access-list bookkeeping): +The `rpc_mainnet` bench runs against **live mainnet state** to validate +real-contract performance (USDC `balanceOf`, a Uniswap V2 `getReserves`). It is +gated behind the `RPC_URL` environment variable and is skipped (not failed) when +it is unset, so `cargo bench` stays offline and CI-reproducible by default: ```sh -cargo bench +RPC_URL=https://eth.llamarpc.com cargo bench --bench rpc_mainnet ``` ## Cargo features -- `protocols` *(default)* — DeFi protocol knowledge: Uniswap V2/V3-style storage - layouts, V3 tick snapshots, and the `inject_v3_*` / `inject_v2_pool_metadata` - helpers. Build with `--no-default-features` for the generic simulation engine - alone (the revert decoder, snapshots/overlays, ERC20 helpers, multicall, deploy, - CREATE3). This surface is slated to move into a separate `evm-amm-state` crate. +| Feature | Default | Gates | +| --- | --- | --- | +| `protocols` | ✅ | DeFi protocol knowledge: Uniswap V2/V3-style storage layouts, V3 tick snapshots, and the `inject_v3_*` / `inject_v2_pool_metadata` helpers. | + +Build with `--no-default-features` for the **generic simulation engine** alone: +the cache core, snapshots/overlays, freshness control plane, access lists, the +revert decoder, ERC20 helpers, multicall, deploy, and CREATE3. The `protocols` +surface is slated to move into a separate `evm-amm-state` crate (see the +[roadmap](docs/ROADMAP.md)); keeping it behind a default feature today lets the +generic core build and lint cleanly without it (CI enforces both configurations). + +## Stability + +`evm-fork-cache` is pre-1.0. Until 1.0, **breaking changes may land in minor +releases** — the roadmap deliberately reshapes the API before the surface +freezes. Each release documents its breaking changes in [`CHANGELOG.md`](CHANGELOG.md). + +- **MSRV:** Rust 1.88 (enforced in CI). Edition 2024. +- **Semver:** pre-1.0 minor versions may break; patch versions will not. +- **Roadmap:** see [`docs/ROADMAP.md`](docs/ROADMAP.md) for the path to 1.0. +- **Known issues / limitations:** see [`docs/KNOWN_ISSUES.md`](docs/KNOWN_ISSUES.md). + +## Contributing + +Contributions are welcome — see [`CONTRIBUTING.md`](CONTRIBUTING.md) for branch +conventions, the green-bar CI expectations, and the commit format. ## License diff --git a/benches/freshness.rs b/benches/freshness.rs index 63a08fa..b60e6f0 100644 --- a/benches/freshness.rs +++ b/benches/freshness.rs @@ -26,6 +26,7 @@ use std::hint::black_box; use std::sync::Arc; use std::time::Duration; +use alloy_eips::BlockId; use alloy_primitives::{Address, Bytes, U256, hex, keccak256}; use alloy_provider::RootProvider; use alloy_provider::network::AnyNetwork; @@ -113,7 +114,7 @@ fn stub_fetcher( values: HashMap<(Address, U256), U256>, delay: Option, ) -> StorageBatchFetchFn { - Arc::new(move |reqs: Vec<(Address, U256)>| { + Arc::new(move |reqs: Vec<(Address, U256)>, _block: Option| { if let Some(d) = delay { std::thread::sleep(d); } @@ -308,5 +309,91 @@ fn bench_phase2_latency(c: &mut Criterion) { group.finish(); } -criterion_group!(benches, bench_phase2_cpu, bench_phase2_latency); +/// Scaling of the `verify_slots` primitive — the background validator's core +/// work — as the volatile set grows (1 → 1000 slots). The (zero-latency) stub +/// reports every slot unchanged, so this isolates the fetch + compare cost from +/// any injection churn. +fn bench_verify_slots(c: &mut Criterion) { + let rt = current_thread_rt(); + let contract = Address::repeat_byte(0xDD); + + let mut group = c.benchmark_group("verify_slots"); + for &n in &[1usize, 10, 100, 1_000] { + let slots: Vec<(Address, U256)> = + (0..n).map(|i| (contract, U256::from(i as u64))).collect(); + let values: HashMap<(Address, U256), U256> = + slots.iter().map(|&key| (key, U256::from(1u64))).collect(); + + let provider = RootProvider::::new(RpcClient::mocked(Asserter::new())); + let mut cache = rt.block_on(EvmCache::new(Arc::new(provider), None)); + // Seed the cached values so the fetched (stub) values match → no change. + let seed: Vec<(Address, U256, U256)> = slots + .iter() + .map(|&(a, s)| (a, s, U256::from(1u64))) + .collect(); + cache.inject_storage_batch(&seed); + cache.set_storage_batch_fetcher(stub_fetcher(values, None)); + + group.throughput(criterion::Throughput::Elements(n as u64)); + group.bench_with_input( + criterion::BenchmarkId::from_parameter(n), + &slots, + |b, slots| { + b.iter(|| { + black_box(cache.verify_slots(slots).unwrap()); + }) + }, + ); + } + group.finish(); +} + +/// Fan-out of the optimistic loop across a batch of K independent sims that all +/// validate as `Confirmed` (stub reports the read slot unchanged). Shows how the +/// per-cycle cost scales with the number of candidate transactions — one frozen +/// snapshot shared across K overlays plus K read-set captures and the unioned +/// verification. +fn bench_multi_sim(c: &mut Criterion) { + let rt = current_thread_rt(); + let calldata = transfer_calldata(1); + + let mut group = c.benchmark_group("multi_sim_confirmed"); + for &k in &[1usize, 4, 16] { + group.throughput(criterion::Throughput::Elements(k as u64)); + group.bench_with_input( + criterion::BenchmarkId::from_parameter(format!("{k}sims")), + &k, + |b, &k| { + b.iter_batched( + || { + let mut cache = swap_cache(&rt, 1_000_000); + cache.set_storage_batch_fetcher(fetcher_for(1_000_000, None)); + let reqs: Vec = (0..k) + .map(|_| SimRequest::new(SENDER, TOKEN, calldata.clone())) + .collect(); + (cache, controller(), reqs) + }, + |(mut cache, mut ctrl, reqs)| { + rt.block_on(async { + let sim = ctrl.run(&mut cache, reqs).unwrap(); + let v = sim.validate().await; + debug_assert!(matches!(v, Validation::Confirmed)); + black_box(v); + }); + }, + BatchSize::SmallInput, + ) + }, + ); + } + group.finish(); +} + +criterion_group!( + benches, + bench_phase2_cpu, + bench_phase2_latency, + bench_verify_slots, + bench_multi_sim +); criterion_main!(benches); diff --git a/benches/rpc_mainnet.rs b/benches/rpc_mainnet.rs new file mode 100644 index 0000000..9f2a9d7 --- /dev/null +++ b/benches/rpc_mainnet.rs @@ -0,0 +1,119 @@ +//! RPC-gated real-contract benchmarks against live forked mainnet state. +//! +//! Unlike the other benches, these fork real chain state, so they are gated +//! behind the `RPC_URL` environment variable and **skip** (rather than fail) +//! when it is unset. This keeps `cargo bench` offline and reproducible by +//! default while still letting you measure real-contract behavior on demand: +//! +//! ```sh +//! RPC_URL=https://eth.llamarpc.com cargo bench --bench rpc_mainnet +//! ``` +//! +//! They measure warm-cache throughput of view calls against well-known mainnet +//! contracts (USDC `balanceOf`, a Uniswap V2 pair `getReserves`). The cache is +//! warmed once before timing so each measured iteration reads from the local +//! cache rather than re-fetching over RPC — that warm-reuse path is exactly what +//! a search loop hammers between block updates. +//! +//! RPC-touching calls run inside `rt.block_on(..)` because `EvmCache` fetches +//! missing state via `tokio::task::block_in_place`, which requires a +//! multi-thread runtime context. + +use std::hint::black_box; +use std::sync::Arc; + +use alloy_primitives::{Address, Bytes, address}; +use alloy_provider::ProviderBuilder; +use alloy_provider::network::AnyNetwork; +use alloy_sol_types::{SolCall, sol}; +use criterion::{Criterion, criterion_group, criterion_main}; +use evm_fork_cache::cache::EvmCache; +use revm::context::result::ExecutionResult; +use revm::primitives::hardfork::SpecId; +use tokio::runtime::Runtime; + +/// USDC (6 decimals) — a ubiquitous mainnet ERC20. +const USDC: Address = address!("A0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48"); +/// A consistently USDC-holding address (an exchange hot wallet). The exact +/// balance is irrelevant to a perf benchmark; `balanceOf` succeeds regardless. +const HOLDER: Address = address!("28C6c06298d514Db089934071355E5743bf21d60"); +/// The Uniswap V2 USDC/WETH pair. +const UNIV2_USDC_WETH: Address = address!("B4e16d0168e52d35CaCD2c6185b44281Ec28C9Dc"); + +sol! { + interface IErc20 { + function balanceOf(address account) external view returns (uint256); + } + interface IUniswapV2Pair { + function getReserves() external view returns (uint112 reserve0, uint112 reserve1, uint32 blockTimestampLast); + } +} + +fn bench_rpc_mainnet(c: &mut Criterion) { + let rpc_url = match std::env::var("RPC_URL") { + Ok(url) if !url.trim().is_empty() => url, + _ => { + eprintln!( + "RPC_URL not set — skipping rpc_mainnet benchmarks. \ + Set RPC_URL= to run them." + ); + return; + } + }; + + // Multi-thread runtime so the cache's lazy fetch (`block_in_place`) is valid. + let rt = Runtime::new().expect("tokio runtime"); + let provider = ProviderBuilder::new() + .network::() + .connect_http(rpc_url.parse().expect("valid RPC_URL")); + let mut cache = rt.block_on( + EvmCache::builder(Arc::new(provider)) + .latest_block() + .spec(SpecId::CANCUN) + .build(), + ); + + let balance_of = Bytes::from(IErc20::balanceOfCall { account: HOLDER }.abi_encode()); + let get_reserves = Bytes::from(IUniswapV2Pair::getReservesCall {}.abi_encode()); + + // Warm the cache once per target so the timed iterations are warm reads. + let warm = rt.block_on(async { + let a = cache.call_raw(HOLDER, USDC, balance_of.clone(), false); + let b = cache.call_raw(Address::ZERO, UNIV2_USDC_WETH, get_reserves.clone(), false); + (a, b) + }); + assert!( + matches!(warm.0, Ok(ExecutionResult::Success { .. })), + "USDC balanceOf warm-up should succeed: {:?}", + warm.0 + ); + assert!( + matches!(warm.1, Ok(ExecutionResult::Success { .. })), + "Uniswap V2 getReserves warm-up should succeed: {:?}", + warm.1 + ); + + let mut group = c.benchmark_group("rpc_mainnet_warm"); + group.bench_function("usdc_balanceOf", |b| { + b.iter(|| { + let r = rt + .block_on(async { cache.call_raw(HOLDER, USDC, balance_of.clone(), false) }) + .unwrap(); + black_box(r); + }) + }); + group.bench_function("univ2_getReserves", |b| { + b.iter(|| { + let r = rt + .block_on(async { + cache.call_raw(Address::ZERO, UNIV2_USDC_WETH, get_reserves.clone(), false) + }) + .unwrap(); + black_box(r); + }) + }); + group.finish(); +} + +criterion_group!(benches, bench_rpc_mainnet); +criterion_main!(benches); diff --git a/benches/simulation.rs b/benches/simulation.rs index b939c4e..10d0d2d 100644 --- a/benches/simulation.rs +++ b/benches/simulation.rs @@ -1,9 +1,16 @@ //! Hot-path benchmarks for the simulation engine: snapshot creation across -//! cache sizes, parallel-overlay fan-out, and batched storage injection. +//! cache sizes, parallel-overlay fan-out, single-call throughput, sequential +//! bundle simulation, and batched storage injection. //! //! These run fully offline (mocked provider) so they're reproducible. They -//! establish the baseline for the Pillar A (copy-on-write snapshot) rewrite — -//! `create_snapshot` is currently an O(total state) deep clone. +//! establish the baseline for the Pillar A (copy-on-write snapshot) rewrite: +//! `create_snapshot` is currently an O(total state) deep clone, so its cost +//! scales with the populated cache size (the `create_snapshot` group sweeps +//! 100 → 10,000 accounts to show that slope). Once Pillar A lands, the same +//! sweep should flatten toward O(changed state) — re-run this group before and +//! after to quantify the win. The `overlay_fanout` group measures the other +//! half of the value proposition: how cheaply one frozen snapshot fans out into +//! many isolated simulations. use std::hint::black_box; use std::sync::Arc; @@ -26,6 +33,7 @@ const BALANCE_SLOT: u64 = 3; sol! { interface MockERC20 { function balanceOf(address account) returns (uint256); + function transfer(address to, uint256 amount) returns (bool); } } @@ -66,8 +74,18 @@ fn populated_cache(rt: &Runtime, accounts: usize, slots_per: usize) -> EvmCache fn bench_create_snapshot(c: &mut Criterion) { let rt = Runtime::new().unwrap(); let mut group = c.benchmark_group("create_snapshot"); - for &(accounts, slots) in &[(100usize, 8usize), (1_000, 8), (2_000, 16)] { + // Sweep from a small pool up to a production-scale index (10k contracts) so + // the O(total state) slope of the current deep clone is visible. Pillar A + // (copy-on-write) should flatten this curve. + for &(accounts, slots) in &[ + (100usize, 8usize), + (1_000, 8), + (2_000, 16), + (5_000, 16), + (10_000, 16), + ] { let cache = populated_cache(&rt, accounts, slots); + group.throughput(criterion::Throughput::Elements((accounts * slots) as u64)); group.bench_with_input( BenchmarkId::from_parameter(format!("{accounts}acct_x{slots}slot")), &cache, @@ -125,22 +143,137 @@ fn bench_overlay_fanout(c: &mut Criterion) { group.finish(); } +/// A cache holding a `MockERC20` with `owner` funded and `recipient` at zero. +fn mock_erc20_cache(rt: &Runtime, token: Address, owner: Address, recipient: Address) -> EvmCache { + let mut cache = offline_cache(rt); + let runtime = Bytecode::new_raw(Bytes::from( + hex::decode(MOCK_ERC20_RUNTIME_HEX.trim()).unwrap(), + )); + let code_hash = runtime.hash_slow(); + cache + .db_mut() + .insert_account_info(Address::ZERO, AccountInfo::default()); + cache + .db_mut() + .insert_account_info(owner, AccountInfo::default()); + cache + .db_mut() + .insert_account_info(recipient, AccountInfo::default()); + cache.db_mut().insert_account_info( + token, + AccountInfo { + balance: U256::ZERO, + nonce: 0, + code: Some(runtime), + code_hash, + account_id: None, + }, + ); + cache + .db_mut() + .replace_account_storage(token, Default::default()) + .unwrap(); + cache + .insert_mapping_storage_slot( + token, + U256::from(BALANCE_SLOT), + owner, + U256::from(1_000_000u64), + ) + .unwrap(); + cache + .insert_mapping_storage_slot(token, U256::from(BALANCE_SLOT), recipient, U256::ZERO) + .unwrap(); + cache +} + +/// Per-call throughput of the primary `EvmCache::call_raw` hot path (a +/// non-committing `balanceOf` view call), warm cache, no RPC. +fn bench_cache_call_raw(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + let token = Address::repeat_byte(0xAA); + let owner = Address::repeat_byte(0xBB); + let recipient = Address::repeat_byte(0xCC); + let mut cache = mock_erc20_cache(&rt, token, owner, recipient); + let calldata = Bytes::from(MockERC20::balanceOfCall { account: owner }.abi_encode()); + + c.bench_function("cache_call_raw/balanceOf", |b| { + b.iter(|| { + let result = cache + .call_raw(owner, token, calldata.clone(), false) + .unwrap(); + debug_assert!(matches!(result, ExecutionResult::Success { .. })); + black_box(result); + }) + }); +} + +/// Sequential bundle: K committing `transfer` calls against shared cache state, +/// the shape of evaluating a multi-step MEV bundle. Measures committed-execution +/// cost as the bundle grows; each iteration starts from a fresh cache so the +/// sender's balance never drains. +fn bench_sim_bundle(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + let token = Address::repeat_byte(0xAA); + let owner = Address::repeat_byte(0xBB); + let recipient = Address::repeat_byte(0xCC); + let calldata = Bytes::from( + MockERC20::transferCall { + to: recipient, + amount: U256::from(1u64), + } + .abi_encode(), + ); + + let mut group = c.benchmark_group("sim_bundle"); + for &k in &[1usize, 4, 16] { + group.throughput(criterion::Throughput::Elements(k as u64)); + group.bench_with_input( + BenchmarkId::from_parameter(format!("{k}tx")), + &k, + |b, &k| { + b.iter_batched( + || mock_erc20_cache(&rt, token, owner, recipient), + |mut cache| { + for _ in 0..k { + let result = cache + .call_raw(owner, token, calldata.clone(), true) + .unwrap(); + black_box(&result); + } + }, + criterion::BatchSize::SmallInput, + ) + }, + ); + } + group.finish(); +} + +/// Batched direct storage injection (the bypass-RPC write path) across sizes. fn bench_inject_storage_batch(c: &mut Criterion) { let rt = Runtime::new().unwrap(); let cache = offline_cache(&rt); - let batch: Vec<(Address, U256, U256)> = (0..1_000) - .map(|i| (addr(i), U256::from(i as u64), U256::from(i as u64))) - .collect(); - c.bench_function("inject_storage_batch/1000", |b| { - b.iter(|| cache.inject_storage_batch(black_box(&batch))) - }); + let mut group = c.benchmark_group("inject_storage_batch"); + for &n in &[100usize, 1_000, 10_000] { + let batch: Vec<(Address, U256, U256)> = (0..n) + .map(|i| (addr(i), U256::from(i as u64), U256::from(i as u64))) + .collect(); + group.throughput(criterion::Throughput::Elements(n as u64)); + group.bench_with_input(BenchmarkId::from_parameter(n), &batch, |b, batch| { + b.iter(|| cache.inject_storage_batch(black_box(batch))) + }); + } + group.finish(); } criterion_group!( benches, bench_create_snapshot, bench_overlay_fanout, + bench_cache_call_raw, + bench_sim_bundle, bench_inject_storage_batch ); criterion_main!(benches); diff --git a/docs/KNOWN_ISSUES.md b/docs/KNOWN_ISSUES.md new file mode 100644 index 0000000..0d93a5d --- /dev/null +++ b/docs/KNOWN_ISSUES.md @@ -0,0 +1,130 @@ +# Known issues & limitations + +A living triage list of bugs, smells, and limitations surfaced during the +publication-readiness review. Items here are **flagged, not fixed** — the test +suite deliberately pins *current* behavior, so changing any of these is a +conscious, reviewable decision (and a `CHANGELOG.md` entry). + +Confidence legend: **[V]** verified against the source during review; +**[R]** reported by the review and worth confirming before acting. + +## Correctness / behavior to review + +1. **[V] Silent persistence failures.** `cache::save_binary_state`, + `PrefetchRegistry::save`, and `ImmutableDataCache::save` log a warning on I/O + error but return `()`, so callers cannot detect a failed write (full disk, + permissions, partial flush). Consider returning `Result<()>` (a breaking + change worth taking pre-1.0). Tested today only insofar as the happy-path + round-trip succeeds. + +2. **[V] Access-list L2 profitability uses an approximate gas model.** In + `access_list.rs`, `into_access_list_if_profitable` / `access_list_if_profitable` + estimate L1 calldata cost with hand-rolled RLP-overhead constants + (`4 * 16` per address, `16` per key, `3 * 16` for the list header). This is an + intentional heuristic, not a precise EIP-2930 serialization cost — verify it + against real serialized sizes before relying on the profitability verdict for + anything other than a rough gate. The two functions also duplicate this logic + (a maintenance hazard: a fix to one must be mirrored). + +3. **[V] Profitability swallows provider errors.** The same functions catch all + provider errors and return `Ok(None)`, which is indistinguishable from + "computed: not profitable." A caller cannot tell a skipped check (RPC down) + from a real negative. Consider a result type that distinguishes the two. + +4. **[R] `set_block` with a tag leaves `block.number` stale.** Only + `BlockId::Number(n)` syncs the `NUMBER` opcode value; pinning to a tag (e.g. + `BlockId::latest()`) leaves the previously-set number in the block env. Either + resolve tags to a concrete number at pin time or document the constraint + loudly. + +5. **[R] Duplicate custom-error selectors shadow silently.** `RevertDecoder` + registration replaces an existing entry for the same 4-byte selector with no + warning, so an accidental double-registration silently wins. Consider a + debug-level log or a `try_register` that reports collisions. + +6. **[R] ERC20 `Transfer` decoding assumes the standard layout.** `inspector.rs` + reads `from`/`to` from indexed topics and `value` from the first 32 data + bytes. Non-standard or packed `Transfer` encodings parse incorrectly. Also, an + address that appears as both `from` and `to` in one transfer is both + subtracted and added (a semantically-invalid self-transfer is not rejected). + +7. **[R] Panic codes above `u64::MAX` are dropped.** `decode_solidity_panic` + converts out-of-range panic codes to `None`. Real compiler-emitted panic codes + are single-byte constants, so this is benign in practice; now documented at the + call site. + +8. **[V] `simulate_call_with_balance_deltas` returns an empty access list.** It + sets `CallSimulationResult.access_list = AccessList::default()`, unlike + `simulate_with_transfer_tracking` which populates it via `extract_access_list`. + Either the field is meaningless on this path or the population was missed — + the docs now state the field is empty here; reconcile before relying on it. + +9. **[V] `call_raw_with_access_list` does not revert its checkpoint on a transact + error.** It propagates the EVM `transact` error with `?` *before* reverting the + journaled checkpoint, whereas `call_raw` / `simulate_with_transfer_tracking` + revert on every path. A host-level transact error therefore leaves the overlay + checkpoint un-reverted. (Reverts normally on success and on revert/halt.) + +10. **[V] `SystemTime::now().unwrap()` panic risk in EVM construction.** + `build_evm` / `make_local_context` (and the overlay equivalents) call + `SystemTime::now().duration_since(UNIX_EPOCH).unwrap()` when no timestamp + override is set, which panics if the system clock is before the Unix epoch. + Setting an explicit timestamp avoids it; consider a saturating fallback. + +## Code-quality nits + +11. **[V] Dead branch in `i128_to_u256`** (`cache/storage_keys.rs`): both the + `value >= 0` and `else` arms evaluate the identical `U256::from(value as u128)`. + The two's-complement cast is correct for both signs, so the `if`/`else` can + collapse to one line (keep the explanatory comment). + +12. **[R] V3 tick-snapshot keys serialize as strings.** `V3PoolTickSnapshot` + stringifies `i16`/`i32` tick/word keys for bincode, then `parse()`s them back + in `to_tick_bitmap`/`to_ticks`, silently dropping any key that fails to parse. + A native integer-keyed encoding would be faster and would not fail silently. + +13. **[V] On-disk caches have no version header.** `binary_state`, `bytecode`, + `metadata` (`ImmutableDataCache`), and `tick_snapshot` all persist raw bincode + with no magic bytes or version field, so a struct-layout change silently + invalidates every existing cache file (decoded as a miss). A version header + would enable detection/migration. + +14. **[R] Balancer pool id keyed by `Debug` formatting.** `ImmutableDataCache` + keys `balancer_pools` by `format!("{:?}", pool_id)`. `Debug` output is not a + stable encoding contract; a hex encoding would be safer for a persisted key. + +## API ergonomics + +15. **[R] `snapshot()` vs `create_snapshot()`.** `snapshot()` returns a low-level + `revm::database::Cache` for in-place `restore()`; `create_snapshot()` returns + an `Arc` for cross-thread fan-out. The names don't convey the + difference. Docs now cross-reference them (see the rustdoc), but a rename + could be considered pre-1.0. + +16. **[R] Process-global cache speed mode.** `set_cache_speed_mode` / + `cache_speed_mode` are a process-wide `static`, so two caches in one process + cannot tune concurrency independently. Phase 1 moved configuration toward + per-instance (`EvmCacheBuilder::cache_config`); the global setter remains. + +17. **[V] `SpeculativeSim` consumption contract.** Both `validate()` and + `into_optimistic()` take `self` by value, so double-consumption is unreachable + under normal ownership. Internally `validate()` uses `.expect("validation + handle taken twice")` (defensive) while `into_optimistic()` no-ops if the + handle was already taken; this is now documented with a `# Panics` note on + `validate`. A `Result`-returning variant could remove the residual foot-gun. + +## Limitations by design / roadmap + +- **No copy-on-write snapshots yet.** `create_snapshot()` deep-clones state + (`O(accounts + slots)`); the COW rewrite is roadmap Pillar A. The `simulation` + benchmarks exist to measure the baseline this will improve on. +- **`protocols` not yet extracted.** The DeFi surface is feature-gated but still + in-crate; `cargo test --no-default-features` is not yet supported because some + unit tests assume the default feature. Extraction into `evm-amm-state` is + planned (roadmap), blocked partly by `ImmutableDataCache` coupling generic + token-decimals with V2/V3/Balancer pool metadata. +- **Event-driven sync (roadmap Pillar B) is not implemented.** Targeted + inject/purge exist; decoding logs into state updates and the WS ingestion loop + with reorg handling are future phases. +- **Recent toolchain.** MSRV 1.88 and edition 2024 are intentional and + CI-enforced; consumers on older toolchains are not supported. diff --git a/examples/foundry_artifact_etching.rs b/examples/foundry_artifact_etching.rs new file mode 100644 index 0000000..c6a91e9 --- /dev/null +++ b/examples/foundry_artifact_etching.rs @@ -0,0 +1,101 @@ +//! Etch a locally compiled Foundry artifact (loaded from a JSON file on disk) +//! over a forked contract, preserving the target's storage, balance, and nonce. +//! +//! This is the on-disk counterpart to `deploy_and_override`: instead of handing +//! raw creation bytecode, you point at a Foundry build artifact +//! (`out/MyContract.sol/MyContract.json`). `etch_foundry_artifact_or_create` +//! reads `bytecode.object`, appends the ABI-encoded constructor args, runs the +//! constructor in the EVM, and copies the resulting runtime bytecode onto the +//! target — the standard way to run a locally-modified contract against forked +//! state. +//! +//! Here the artifact is the checked-in `fixtures/MockERC20.foundry.json` (a +//! minimal Foundry-shaped artifact wrapping the `MockERC20` creation bytecode). +//! It is etched over a target that already holds a token balance, and we show +//! that balance survives the code swap. +//! +//! Runs fully offline against a mocked provider. +//! +//! Run with: +//! +//! ```sh +//! cargo run --example foundry_artifact_etching +//! ``` + +use alloy_primitives::{Address, U256}; +use anyhow::Result; +use evm_fork_cache::deploy::{encode_constructor_args, etch_foundry_artifact_or_create}; + +#[path = "support/mock.rs"] +mod mock; + +/// Path to the checked-in Foundry artifact (resolved relative to the crate root +/// so the example runs from any working directory). +const ARTIFACT: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/fixtures/MockERC20.foundry.json" +); + +/// Deterministic CREATE address for `Address::ZERO` at nonce 0 (the scratch +/// address the artifact is deployed to before being etched onto the target). +const CREATE_ADDRESS_ZERO_NONCE_0: Address = Address::new(alloy_primitives::hex!( + "bd770416a3345f91e4b34576cb804a576fa48eb1" +)); + +#[tokio::main(flavor = "multi_thread")] +async fn main() -> Result<()> { + let mut cache = mock::offline_cache().await?; + mock::install_default_account(&mut cache, Address::ZERO); + // Pre-insert the scratch CREATE address so the mocked provider is never queried. + mock::install_default_account(&mut cache, CREATE_ADDRESS_ZERO_NONCE_0); + + // A target that already holds storage on the fork (a holder balance). + let target = Address::repeat_byte(0xCC); + let holder = Address::repeat_byte(0xDD); + mock::install_mock_erc20(&mut cache, target); + mock::install_default_account(&mut cache, holder); + cache.insert_mapping_storage_slot( + target, + U256::from(mock::MOCK_ERC20_BALANCE_SLOT), + holder, + U256::from(7_777u64), + )?; + println!( + "target {target} holder balance (before etch): {}", + mock::balance_of(&mut cache, target, holder)? + ); + + // Constructor args for MockERC20(string name, string symbol, uint8 decimals). + let constructor_args = encode_constructor_args(( + String::from("Etched Token"), + String::from("ETCH"), + U256::from(18u8), + )); + + // Load the artifact from disk, deploy it, and etch its runtime code onto the + // target. Only the bytecode is replaced; the target's storage is preserved. + let etched = etch_foundry_artifact_or_create( + &mut cache, + target, + ARTIFACT, + Address::ZERO, + constructor_args, + )?; + + println!( + "etched {} bytes from {} over {}", + etched.code_size, etched.deployed_address, etched.target_address, + ); + println!( + "target holder balance (after etch): {} (storage preserved)", + mock::balance_of(&mut cache, target, holder)? + ); + + assert_eq!( + mock::balance_of(&mut cache, target, holder)?, + U256::from(7_777u64), + "etching runtime bytecode must preserve the target's storage" + ); + + Ok(()) +} diff --git a/examples/freshness_multi_sim.rs b/examples/freshness_multi_sim.rs new file mode 100644 index 0000000..15a8780 --- /dev/null +++ b/examples/freshness_multi_sim.rs @@ -0,0 +1,166 @@ +//! Many optimistic sims at once: only the sim whose state actually changed is +//! re-run, and `ValidThrough` classification ages a slot from pinned to volatile. +//! +//! This builds on `freshness_optimistic` (read that first). Three independent +//! `transfer` sims run against one frozen snapshot. A stub fetcher then reports +//! that **only the second sender's** balance has dropped below its transfer +//! amount. The background validator therefore re-runs **only that one sim** (the +//! others' read-sets were unaffected), so the `Corrected` verdict carries a +//! single changed slot and a single re-executed result. +//! +//! It also shows the classification layer: one slot is `Pinned` (never +//! verified), and one is `ValidThrough(block)` — pinned until a target block, +//! then volatile. Advancing the controller's block clock past that block ages it +//! into the volatile set. +//! +//! Runs fully offline against a mocked provider and a stubbed +//! `StorageBatchFetchFn`; no network access. +//! +//! Run with: +//! +//! ```sh +//! cargo run --example freshness_multi_sim +//! ``` + +use std::collections::HashMap; +use std::sync::Arc; + +use alloy_eips::BlockId; +use alloy_primitives::{Address, Bytes, U256, keccak256}; +use alloy_sol_types::{SolCall, SolValue}; +use anyhow::Result; +use evm_fork_cache::cache::StorageBatchFetchFn; +use evm_fork_cache::freshness::{ + AlwaysVerify, FreshnessController, FreshnessRegistry, SimRequest, Validation, Validity, +}; + +#[path = "support/mock.rs"] +mod mock; + +/// Hashed storage slot of `balanceOf[owner]` (mapping at slot 3). +fn balance_slot(owner: Address) -> U256 { + let key = keccak256((owner, U256::from(mock::MOCK_ERC20_BALANCE_SLOT)).abi_encode()); + U256::from_be_bytes(key.0) +} + +fn transfer_calldata(to: Address, amount: u64) -> Bytes { + Bytes::from( + mock::MockERC20::transferCall { + to, + amount: U256::from(amount), + } + .abi_encode(), + ) +} + +#[tokio::main(flavor = "multi_thread")] +async fn main() -> Result<()> { + let mut cache = mock::offline_cache().await?; + let token = Address::repeat_byte(0x11); + mock::install_default_account(&mut cache, Address::ZERO); + mock::install_mock_erc20(&mut cache, token); + + // Three senders, each funded 1000, each transferring 100 to a distinct + // recipient so their read-sets are disjoint. + let senders = [ + Address::repeat_byte(0xA1), + Address::repeat_byte(0xB2), + Address::repeat_byte(0xC3), + ]; + let recipients = [ + Address::repeat_byte(0x5A), + Address::repeat_byte(0x5B), + Address::repeat_byte(0x5C), + ]; + for s in &senders { + mock::install_default_account(&mut cache, *s); + cache.inject_storage_batch(&[(token, balance_slot(*s), U256::from(1000))]); + } + + // Stub fetcher: every sender's balance is unchanged EXCEPT the second, whose + // fresh balance has dropped to 50 — too small to cover its transfer of 100. + let fresh: HashMap<(Address, U256), U256> = HashMap::from([ + ((token, balance_slot(senders[0])), U256::from(1000)), + ((token, balance_slot(senders[1])), U256::from(50)), // changed! + ((token, balance_slot(senders[2])), U256::from(1000)), + ]); + let fetcher: StorageBatchFetchFn = Arc::new( + move |requests: Vec<(Address, U256)>, _block: Option| { + requests + .into_iter() + .map(|(addr, slot)| { + let v = fresh.get(&(addr, slot)).copied().unwrap_or(U256::ZERO); + (addr, slot, Ok(v)) + }) + .collect() + }, + ); + cache.set_storage_batch_fetcher(fetcher); + + // ── Classification layer ─────────────────────────────────────────────── + // Slot 6 is treated as immutable (Pinned, never verified). Slot 7 is valid + // through block 100, then becomes volatile. + let mut registry = FreshnessRegistry::new(); + registry.pin_slot(token, U256::from(6)); + registry.valid_through_slot(token, U256::from(7), 100); + + println!("classification:"); + println!( + " pinned slot 6 volatile? {} (never)", + registry.is_volatile(token, U256::from(6), 100) + ); + println!( + " valid-through(100) slot 7 at block 100: volatile? {} (still valid)", + registry.is_volatile(token, U256::from(7), 100) + ); + println!( + " valid-through(100) slot 7 at block 101: volatile? {} (aged into volatile)\n", + registry.is_volatile(token, U256::from(7), 101) + ); + debug_assert_eq!(registry.validity(token, U256::from(6)), Validity::Pinned); + + // ── Optimistic multi-sim run ─────────────────────────────────────────── + let mut controller = FreshnessController::new(registry, AlwaysVerify); + let requests: Vec = senders + .iter() + .zip(recipients.iter()) + .map(|(&from, &to)| SimRequest::new(from, token, transfer_calldata(to, 100))) + .collect(); + + let sim = controller.run(&mut cache, requests)?; + + // All three optimistic transfers succeed against the 1000-balance snapshot. + let optimistic_ok: Vec = sim + .optimistic() + .iter() + .map(|r| !r.logs.is_empty()) + .collect(); + println!("optimistic (against the snapshot): {optimistic_ok:?} (all succeed)"); + + match sim.validate().await { + Validation::Corrected { results, changed } => { + println!( + "\nvalidation: Corrected — {} slot(s) changed:", + changed.len() + ); + for c in &changed { + println!(" sender slot {} : {} -> {}", c.slot, c.old, c.new); + } + let corrected_ok: Vec = results.iter().map(|r| !r.logs.is_empty()).collect(); + println!("corrected results: {corrected_ok:?}"); + + // Exactly the second sim flipped success -> revert; the others are + // untouched (selective re-run). + assert_eq!(changed.len(), 1, "only one sender's balance changed"); + assert_eq!(corrected_ok, vec![true, false, true]); + println!( + "\n→ only sim #2 was re-run (its balance fell below the transfer); \ + sims #1 and #3 were left as-is." + ); + } + Validation::Confirmed => println!("validation: Confirmed (unexpected here)"), + Validation::Unverified { reason } => println!("validation: Unverified — {reason}"), + } + + Ok(()) +} diff --git a/examples/freshness_optimistic.rs b/examples/freshness_optimistic.rs index 274315d..42fa191 100644 --- a/examples/freshness_optimistic.rs +++ b/examples/freshness_optimistic.rs @@ -23,10 +23,11 @@ use std::collections::HashMap; use std::sync::Arc; +use alloy_eips::BlockId; use alloy_primitives::{Address, Bytes, U256, keccak256}; use alloy_sol_types::{SolCall, SolValue}; use anyhow::Result; -use evm_fork_cache::cache::StorageBatchFetchFn; +use evm_fork_cache::cache::{SimStatus, StorageBatchFetchFn}; use evm_fork_cache::freshness::{ AlwaysVerify, FreshnessController, FreshnessRegistry, SimRequest, Validation, }; @@ -60,15 +61,17 @@ async fn main() -> Result<()> { // (An unmapped slot reads as zero, matching how a sim reads an unseen slot.) let fresh: HashMap<(Address, U256), U256> = HashMap::from([((token, owner_slot), U256::from(50))]); - let fetcher: StorageBatchFetchFn = Arc::new(move |requests: Vec<(Address, U256)>| { - requests - .into_iter() - .map(|(addr, slot)| { - let value = fresh.get(&(addr, slot)).copied().unwrap_or(U256::ZERO); - (addr, slot, Ok(value)) - }) - .collect() - }); + let fetcher: StorageBatchFetchFn = Arc::new( + move |requests: Vec<(Address, U256)>, _block: Option| { + requests + .into_iter() + .map(|(addr, slot)| { + let value = fresh.get(&(addr, slot)).copied().unwrap_or(U256::ZERO); + (addr, slot, Ok(value)) + }) + .collect() + }, + ); cache.set_storage_batch_fetcher(fetcher); // Classification: the balance slot is volatile (default), and slot 6 (a @@ -92,7 +95,7 @@ async fn main() -> Result<()> { let sim = controller.run(&mut cache, vec![request])?; let optimistic = &sim.optimistic()[0]; - let optimistic_succeeded = !optimistic.logs.is_empty(); + let optimistic_succeeded = matches!(optimistic.status, SimStatus::Success); println!("optimistic result (computed immediately, against the snapshot):"); println!(" gas_used = {}", optimistic.gas_used); println!( @@ -116,7 +119,7 @@ async fn main() -> Result<()> { println!(" {} slot {} : {} -> {}", c.address, c.slot, c.old, c.new); } let corrected = &results[0]; - let corrected_succeeded = !corrected.logs.is_empty(); + let corrected_succeeded = matches!(corrected.status, SimStatus::Success); println!( "\ncorrected re-run: gas_used = {}, transfer {} (emitted {} log(s))", corrected.gas_used, diff --git a/examples/multi_hop_swap.rs b/examples/multi_hop_swap.rs new file mode 100644 index 0000000..0eb07a5 --- /dev/null +++ b/examples/multi_hop_swap.rs @@ -0,0 +1,88 @@ +//! Simulate a multi-hop Uniswap V2 swap quote against live mainnet state. +//! +//! This calls the real Uniswap V2 router's `getAmountsOut(amountIn, path)` for a +//! two-hop path (WETH → USDC → DAI) inside the fork. The router reads each pair's +//! reserves from chain state — fetched lazily through the cache on first access — +//! and returns the output amount after both hops. It is a pure view call, so no +//! funding or approvals are needed, yet it exercises the real multi-contract +//! state a swap simulation depends on. +//! +//! To go further (a state-changing swap), you would override the caller's input +//! token balance (see `fork_override_balance`) and call the router's +//! `swapExactTokensForTokens`, then read the balance deltas with +//! `simulate_with_transfer_tracking`. +//! +//! Requires an Ethereum mainnet RPC endpoint. Run with: +//! +//! ```sh +//! RPC_URL=https://eth.llamarpc.com cargo run --example multi_hop_swap +//! ``` + +use std::sync::Arc; + +use alloy_eips::BlockId; +use alloy_primitives::{Address, Bytes, U256, address}; +use alloy_provider::ProviderBuilder; +use alloy_provider::network::AnyNetwork; +use alloy_sol_types::{SolCall, sol}; +use anyhow::{Result, anyhow}; +use evm_fork_cache::cache::EvmCache; +use revm::context::result::ExecutionResult; + +const ROUTER: Address = address!("7a250d5630B4cF539739dF2C5dAcb4c659F2488D"); +const WETH: Address = address!("C02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2"); +const USDC: Address = address!("A0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48"); +const DAI: Address = address!("6B175474E89094C44Da98b954EedeAC495271d0F"); + +sol! { + interface IUniswapV2Router { + function getAmountsOut(uint256 amountIn, address[] path) external view returns (uint256[] amounts); + } +} + +#[tokio::main(flavor = "multi_thread")] +async fn main() -> Result<()> { + let Ok(rpc_url) = std::env::var("RPC_URL") else { + eprintln!("This example needs an Ethereum mainnet RPC endpoint. Run with:"); + eprintln!(" RPC_URL=https://eth.llamarpc.com cargo run --example multi_hop_swap"); + return Ok(()); + }; + + let provider = ProviderBuilder::new() + .network::() + .connect_http(rpc_url.parse()?); + let mut cache = EvmCache::new(Arc::new(provider), Some(BlockId::latest())).await; + + // Quote 1 WETH swapped along WETH -> USDC -> DAI. + let amount_in = U256::from(10u64).pow(U256::from(18u64)); // 1 WETH (1e18) + let path = vec![WETH, USDC, DAI]; + let calldata = Bytes::from( + IUniswapV2Router::getAmountsOutCall { + amountIn: amount_in, + path: path.clone(), + } + .abi_encode(), + ); + + let result = cache.call_raw(Address::ZERO, ROUTER, calldata, false)?; + let output = match result { + ExecutionResult::Success { output, .. } => output.into_data(), + other => return Err(anyhow!("getAmountsOut did not succeed: {other:?}")), + }; + + let amounts = IUniswapV2Router::getAmountsOutCall::abi_decode_returns(&output)?; + if amounts.len() != path.len() { + return Err(anyhow!("unexpected amounts length: {}", amounts.len())); + } + + // USDC has 6 decimals, DAI has 18; print human-readable figures. + let usdc_mid = amounts[1] / U256::from(10u64).pow(U256::from(6u64)); + let dai_out = amounts[2] / U256::from(10u64).pow(U256::from(18u64)); + + println!("two-hop quote (Uniswap V2, live reserves):"); + println!(" in: 1 WETH"); + println!(" hop1 -> ~{usdc_mid} USDC ({} raw)", amounts[1]); + println!(" hop2 -> ~{dai_out} DAI ({} raw)", amounts[2]); + + Ok(()) +} diff --git a/examples/multicall_with_error_handling.rs b/examples/multicall_with_error_handling.rs new file mode 100644 index 0000000..7eabffd --- /dev/null +++ b/examples/multicall_with_error_handling.rs @@ -0,0 +1,110 @@ +//! Batch calls with `allowFailure` and read partial results. +//! +//! Multicall3's `aggregate3` lets each call opt into failure tolerance. With +//! `allow_failure = true`, a call that reverts does **not** abort the batch — +//! it comes back with `success = false` and whatever revert data it produced, +//! so a search loop can probe many calls in one pass and gracefully skip the +//! ones that fail. (With `allow_failure = false`, a revert makes the whole +//! `aggregate3` revert, surfacing here as an `Err` from `execute`.) +//! +//! This batch mixes calls that succeed (`USDC.decimals()`, `USDC.balanceOf(..)`) +//! with one that reverts (`USDC.transfer(..)` from the Multicall3 contract, which +//! holds no USDC). `try_decode_result` returns `None` for the failed call instead +//! of erroring. +//! +//! Requires an Ethereum mainnet RPC endpoint. Run with: +//! +//! ```sh +//! RPC_URL=https://eth.llamarpc.com cargo run --example multicall_with_error_handling +//! ``` + +use std::sync::Arc; + +use alloy_eips::BlockId; +use alloy_primitives::{Address, U256, address}; +use alloy_provider::ProviderBuilder; +use alloy_provider::network::AnyNetwork; +use alloy_sol_types::sol; +use anyhow::Result; +use evm_fork_cache::cache::EvmCache; +use evm_fork_cache::multicall::{MulticallBatch, try_decode_result}; + +const USDC: Address = address!("A0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48"); +const HOLDER: Address = address!("28C6c06298d514Db089934071355E5743bf21d60"); + +sol! { + interface IUsdc { + function decimals() external view returns (uint8); + function balanceOf(address account) external view returns (uint256); + function transfer(address to, uint256 amount) external returns (bool); + } +} + +#[tokio::main(flavor = "multi_thread")] +async fn main() -> Result<()> { + let Ok(rpc_url) = std::env::var("RPC_URL") else { + eprintln!("This example needs an Ethereum mainnet RPC endpoint. Run with:"); + eprintln!( + " RPC_URL=https://eth.llamarpc.com cargo run --example multicall_with_error_handling" + ); + return Ok(()); + }; + + let provider = ProviderBuilder::new() + .network::() + .connect_http(rpc_url.parse()?); + let mut cache = EvmCache::new(Arc::new(provider), Some(BlockId::latest())).await; + + // Three calls, all failure-tolerant. The transfer reverts (the Multicall3 + // contract — the msg.sender of each sub-call — holds no USDC), but the batch + // still completes and the other two results are usable. + let mut batch = MulticallBatch::with_capacity(3); + batch.add_call(USDC, IUsdc::decimalsCall {}, true); + batch.add_call(USDC, IUsdc::balanceOfCall { account: HOLDER }, true); + batch.add_call( + USDC, + IUsdc::transferCall { + to: HOLDER, + amount: U256::MAX, + }, + true, + ); + + let results = batch.execute(&mut cache)?; + println!( + "batch of {} calls completed despite a revert:\n", + results.len() + ); + + let decimals = try_decode_result::(&results[0]); + let balance = try_decode_result::(&results[1]); + + println!( + " [0] decimals() success={} -> {:?}", + results[0].success, decimals + ); + println!( + " [1] balanceOf(holder) success={} -> {:?}", + results[1].success, balance + ); + println!( + " [2] transfer(.., MAX) success={} -> {} (gracefully skipped)", + results[2].success, + if results[2].success { + "ok" + } else { + "reverted, no value" + } + ); + + assert!( + results[0].success && results[1].success, + "view calls succeed" + ); + assert!( + !results[2].success, + "the unfunded transfer reverts but does not abort the batch" + ); + + Ok(()) +} diff --git a/fixtures/MockERC20.foundry.json b/fixtures/MockERC20.foundry.json new file mode 100644 index 0000000..cc5644d --- /dev/null +++ b/fixtures/MockERC20.foundry.json @@ -0,0 +1,26 @@ +{ + "_comment": "Minimal Foundry-style build artifact for the MockERC20 fixture (see MockERC20.sol). Only `bytecode.object` is consumed by load_foundry_creation_code; the other fields mirror Foundry's `out/*.json` layout for realism.", + "abi": [ + { + "type": "constructor", + "inputs": [ + { + "name": "_name", + "type": "string" + }, + { + "name": "_symbol", + "type": "string" + }, + { + "name": "_decimals", + "type": "uint8" + } + ], + "stateMutability": "nonpayable" + } + ], + "bytecode": { + "object": "0x60a060405234610341576109f88038038061001981610345565b9283398101906060818303126103415780516001600160401b038111610341578261004591830161036a565b60208201519092906001600160401b0381116103415760409161006991840161036a565b91015160ff811681036103415782516001600160401b03811161024a575f54600181811c91168015610337575b602082101461022c57601f81116102ca575b506020601f821160011461026957819293945f9261025e575b50508160011b915f199060031b1c1916175f555b81516001600160401b03811161024a57600154600181811c91168015610240575b602082101461022c57601f81116101be575b50602092601f821160011461015d57928192935f92610152575b50508160011b915f199060031b1c1916176001555b60805260405161063c90816103bc8239608051816102c90152f35b015190505f80610122565b601f1982169360015f52805f20915f5b8681106101a6575083600195961061018e575b505050811b01600155610137565b01515f1960f88460031b161c191690555f8080610180565b9192602060018192868501518155019401920161016d565b818111156101085760015f52601f820160051c7fb10e2d527612073b26eecdfd717e6a320cf44b4afac2b0732d9fcbe2b7fa0cf660208410610224575b81601f9101920160051c03905f5b828110610217575050610108565b5f82820155600101610209565b5f91506101fb565b634e487b7160e01b5f52602260045260245ffd5b90607f16906100f6565b634e487b7160e01b5f52604160045260245ffd5b015190505f806100c1565b601f198216905f8052805f20915f5b8181106102b25750958360019596971061029a575b505050811b015f556100d5565b01515f1960f88460031b161c191690555f808061028d565b9192602060018192868b015181550194019201610278565b818111156100a8575f8052601f820160051c7f290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e5636020841061032f575b81601f9101920160051c03905f5b8281106103225750506100a8565b5f82820155600101610314565b5f9150610306565b90607f1690610096565b5f80fd5b6040519190601f01601f191682016001600160401b0381118382101761024a57604052565b81601f82011215610341578051906001600160401b03821161024a57610399601f8301601f1916602001610345565b928284526020838301011161034157815f9260208093018386015e830101529056fe60806040526004361015610011575f80fd5b5f3560e01c806306fdde03146103ff578063095ea7b3146103b857806318160ddd1461039b57806323b872dd146102ed578063313ce567146102b05780634e6ec2471461026257806370a082311461022a57806395d89b411461010c578063a9059cbb146100db5763dd62ed3e14610087575f80fd5b346100d75760403660031901126100d7576100a06104fb565b6100a8610511565b6001600160a01b039182165f908152600460209081526040808320949093168252928352819020549051908152f35b5f80fd5b346100d75760403660031901126100d7576101016100f76104fb565b6024359033610555565b602060405160018152f35b346100d7575f3660031901126100d7576040515f6001548060011c90600181168015610220575b60208310811461020c578285529081156101f0575060011461019a575b50819003601f01601f191681019067ffffffffffffffff821181831017610186576040829052819061018290826104d1565b0390f35b634e487b7160e01b5f52604160045260245ffd5b60015f9081529091507fb10e2d527612073b26eecdfd717e6a320cf44b4afac2b0732d9fcbe2b7fa0cf65b8282106101da57506020915082010182610150565b60018160209254838588010152019101906101c5565b90506020925060ff191682840152151560051b82010182610150565b634e487b7160e01b5f52602260045260245ffd5b91607f1691610133565b346100d75760203660031901126100d7576001600160a01b0361024b6104fb565b165f526003602052602060405f2054604051908152f35b346100d75760403660031901126100d75761027b6104fb565b6024359061028b82600254610548565b60025560018060a01b03165f5260036020526102ac60405f20918254610548565b9055005b346100d7575f3660031901126100d757602060405160ff7f0000000000000000000000000000000000000000000000000000000000000000168152f35b346100d75760603660031901126100d7576103066104fb565b61030e610511565b6001600160a01b0382165f81815260046020908152604080832033845290915290205492604435929183851061036a5761034b8461010196610527565b5f91825260046020908152604080842033855290915290912055610555565b60405162461bcd60e51b8152602060048201526009602482015268616c6c6f77616e636560b81b6044820152606490fd5b346100d7575f3660031901126100d7576020600254604051908152f35b346100d75760403660031901126100d7576103d16104fb565b335f52600460205260405f209060018060a01b03165f5260205260405f206024359055602060405160018152f35b346100d7575f3660031901126100d7576040515f5f548060011c906001811680156104c7575b60208310811461020c578285529081156101f057506001146104735750819003601f01601f191681019067ffffffffffffffff821181831017610186576040829052819061018290826104d1565b5f8080529091507f290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e5635b8282106104b157506020915082010182610150565b600181602092548385880101520191019061049c565b91607f1691610425565b602060409281835280519182918282860152018484015e5f828201840152601f01601f1916010190565b600435906001600160a01b03821682036100d757565b602435906001600160a01b03821682036100d757565b9190820391821161053457565b634e487b7160e01b5f52601160045260245ffd5b9190820180921161053457565b60018060a01b031690815f5260036020528260405f2054106105d75760207fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef91835f526003825260405f206105ab868254610527565b905560018060a01b031693845f526003825260405f206105cc828254610548565b9055604051908152a3565b60405162461bcd60e51b815260206004820152600760248201526662616c616e636560c81b6044820152606490fdfea26469706673582212204442ecaa121ad6d723e2058cae7a9c3d491ebfca3a749fbe07f0b26379ec2b9064736f6c63430008220033" + } +} diff --git a/fixtures/README.md b/fixtures/README.md index 619ee89..ce6f067 100644 --- a/fixtures/README.md +++ b/fixtures/README.md @@ -14,6 +14,10 @@ deployment helpers without touching a real network. token directly at an address via `db_mut().insert_account_info`. - `mock_erc20_creation.hex` — creation bytecode, for `deploy_contract`. The constructor takes `(string name, string symbol, uint8 decimals)`. +- `MockERC20.foundry.json` — a minimal Foundry-shaped build artifact wrapping the + creation bytecode in `bytecode.object`, used by the `foundry_artifact_etching` + example to exercise `deploy::etch_foundry_artifact*` (which load from a JSON + artifact on disk). Regenerated from `mock_erc20_creation.hex`. ### Storage layout diff --git a/src/access_list.rs b/src/access_list.rs index f0b9353..a84c99e 100644 --- a/src/access_list.rs +++ b/src/access_list.rs @@ -26,6 +26,10 @@ use crate::cache::EvmCache; const ARB_GAS_INFO: Address = address!("000000000000000000000000000000000000006C"); /// Optimism GasPriceOracle predeploy (Bedrock+). +/// +/// Fixed predeploy address on every OP Stack chain. Queried for the L1 base fee +/// ([`query_l1_base_fee_for_chain`]) and the full Ecotone L1 data fee +/// ([`compute_op_l1_fee`]). pub const OP_GAS_PRICE_ORACLE: Address = address!("420000000000000000000000000000000000000F"); /// Chain fee model used when deciding whether an access list is worth posting. @@ -70,11 +74,20 @@ pub struct SmartAccessList { impl SmartAccessList { /// Create an empty smart access-list builder. + /// + /// Populate it with [`SmartAccessList::add_address`] and + /// [`SmartAccessList::add_storage_key`], then finalize with one of the + /// `into_access_list_*` methods. pub fn new() -> Self { Self { items: Vec::new() } } /// Create a builder from precomputed EIP-2930 items. + /// + /// The items are taken as-is; this constructor does not deduplicate + /// addresses or storage keys (unlike [`SmartAccessList::add_address`] and + /// [`SmartAccessList::add_storage_key`]). Pass items that are already + /// distinct, or rely on downstream encoders to fold duplicates. pub fn from_items(items: Vec) -> Self { Self { items } } @@ -121,11 +134,43 @@ impl SmartAccessList { /// Evaluate profitability against current L1/L2 gas prices and return /// the access list only if it saves money. /// - /// Queries the ArbGasInfo precompile for pricing, then compares the - /// L2 execution savings (100 gas per entry) against the L1 data cost - /// of serializing each entry. + /// Queries the Arbitrum `ArbGasInfo` precompile for pricing, then compares + /// the L2 execution savings against the estimated L1 data cost of posting + /// the serialized list: + /// + /// - **L2 savings**: `100 gas * entry_count * perArbGas`, where each address + /// and each storage key counts as one entry (the EIP-2929 warm-vs-cold + /// access discount). + /// - **L1 cost**: `l1_data_gas * l1_base_fee`, where `l1_data_gas` sums the + /// per-byte calldata gas ([`l1_data_gas_for_bytes`]) of every address and + /// key plus a fixed RLP-framing surcharge. + /// + /// # Cost model is approximate /// - /// Returns `Ok(None)` if unprofitable or on pricing query failure. + /// The RLP-overhead constants — roughly `4 * 16` gas per address entry, + /// `16` gas per storage key, and `3 * 16` gas for the top-level list headers + /// — are a deliberate **approximation**, not the exact EIP-2930 RLP + /// serialization cost. They assume worst-case non-zero framing bytes and do + /// not account for the real RLP length-prefix sizing, address/key sharing, + /// or rollup-specific compression. Treat this as a rough profitability gate, + /// not a precise gas accounting: a list near the break-even point may be + /// classified either way. + /// + /// # Errors + /// + /// Returns `Err` only if the call wrapper itself surfaces a non-recoverable + /// error; in practice provider/pricing failures do **not** error. + /// + /// Returns `Ok(None)` when: + /// - the list is empty, + /// - the `ArbGasInfo` pricing or L1-base-fee query fails (the error is logged + /// at `debug` and swallowed — see below), + /// - either the L2 or L1 gas price reads as zero, or + /// - the estimated L1 cost meets or exceeds the L2 savings (not profitable). + /// + /// A `None` returned because a provider query failed is **indistinguishable** + /// from a `None` returned because the list was genuinely unprofitable: both + /// surface as a skipped access list, not as an error. pub async fn into_access_list_if_profitable( self, provider: &P, @@ -212,7 +257,35 @@ impl SmartAccessList { /// but costs L1 data posting gas for its serialized bytes. This function /// computes the net and returns the access list only if profitable. /// -/// Returns `Ok(None)` if the list is empty, unprofitable, or pricing queries fail. +/// This is the free-function counterpart to +/// [`SmartAccessList::into_access_list_if_profitable`] for a pre-built +/// [`AccessList`]; the two share the same cost model and break-even comparison. +/// +/// # Cost model is approximate +/// +/// As with [`SmartAccessList::into_access_list_if_profitable`], the L1 cost is +/// estimated from per-byte calldata gas ([`l1_data_gas_for_bytes`]) plus fixed +/// RLP-framing surcharges (`4 * 16` gas per address, `16` gas per key, `3 * 16` +/// gas for the top-level headers). Those framing constants are an +/// **approximation**, not the exact EIP-2930 RLP serialization cost: they assume +/// worst-case non-zero bytes and ignore real length-prefix sizing and +/// rollup-specific compression. Treat the result as a rough profitability gate. +/// +/// # Errors +/// +/// Returns `Err` only if the call wrapper itself surfaces a non-recoverable +/// error; in practice provider/pricing failures do **not** error. +/// +/// Returns `Ok(None)` when: +/// - the list is empty, +/// - the `ArbGasInfo` pricing or L1-base-fee query fails (the error is logged at +/// `debug` and swallowed), +/// - either the L2 or L1 gas price reads as zero, or +/// - the estimated L1 cost meets or exceeds the L2 savings (not profitable). +/// +/// A `None` returned because a provider query failed is **indistinguishable** +/// from a `None` returned because the list was genuinely unprofitable: both +/// surface as a skipped access list, not as an error. pub async fn access_list_if_profitable( access_list: AccessList, provider: &P, @@ -387,6 +460,23 @@ fn push_unique(vec: &mut Vec, val: B256) { } /// L1 calldata gas for a byte slice: zero bytes = 4 gas, non-zero = 16 gas. +/// +/// This is the post-EIP-2028 calldata pricing used to approximate the L1 data +/// cost of serialized access-list entries. It counts the raw bytes only and +/// does not add any RLP framing overhead. +/// +/// # Examples +/// +/// ``` +/// use evm_fork_cache::access_list::l1_data_gas_for_bytes; +/// +/// // All-zero 32-byte slot: 32 * 4 = 128 gas. +/// assert_eq!(l1_data_gas_for_bytes(&[0u8; 32]), 128); +/// // All-non-zero 20-byte address: 20 * 16 = 320 gas. +/// assert_eq!(l1_data_gas_for_bytes(&[0xFFu8; 20]), 320); +/// // Empty slice costs nothing. +/// assert_eq!(l1_data_gas_for_bytes(&[]), 0); +/// ``` pub fn l1_data_gas_for_bytes(data: &[u8]) -> u64 { data.iter() .map(|&b| if b == 0 { 4u64 } else { 16u64 }) diff --git a/src/access_set.rs b/src/access_set.rs index 187cbd0..3e511f0 100644 --- a/src/access_set.rs +++ b/src/access_set.rs @@ -39,6 +39,41 @@ impl StorageAccessList { self.slots.len() } + /// Merge another touch set into this one (set union of accounts and slots). + /// + /// Duplicate accounts and `(account, slot)` pairs already present are not + /// counted twice, so [`StorageAccessList::account_count`] and + /// [`StorageAccessList::slot_count`] reflect distinct entries after merging. + /// + /// # Examples + /// + /// ``` + /// use evm_fork_cache::StorageAccessList; + /// use alloy_primitives::{Address, U256}; + /// + /// let acct_a = Address::repeat_byte(0x01); + /// let acct_b = Address::repeat_byte(0x02); + /// + /// let mut base = StorageAccessList::default(); + /// base.accounts.insert(acct_a); + /// base.slots.insert((acct_a, U256::from(1))); + /// + /// let mut other = StorageAccessList::default(); + /// other.accounts.insert(acct_a); // overlaps `base`, not double-counted + /// other.accounts.insert(acct_b); + /// other.slots.insert((acct_b, U256::from(2))); + /// + /// base.extend(&other); + /// + /// assert_eq!(base.account_count(), 2); + /// assert_eq!(base.slot_count(), 2); + /// assert!(!base.is_empty()); + /// ``` + pub fn extend(&mut self, other: &Self) { + self.accounts.extend(&other.accounts); + self.slots.extend(&other.slots); + } + /// Compute EIP-2929 gas saved when this touch set runs after `warm`. /// /// Cold account access costs 2600 gas versus 100 gas when warm, saving @@ -50,12 +85,6 @@ impl StorageAccessList { shared_accounts * 2500 + shared_slots * 2000 } - /// Merge another touch set into this one. - pub fn extend(&mut self, other: &Self) { - self.accounts.extend(&other.accounts); - self.slots.extend(&other.slots); - } - /// Convert this touch set into an EIP-2930 transaction access list. pub fn to_eip2930(&self) -> AccessList { let mut by_address: std::collections::BTreeMap> = self diff --git a/src/cache/binary_state.rs b/src/cache/binary_state.rs index a9aa74d..1deef61 100644 --- a/src/cache/binary_state.rs +++ b/src/cache/binary_state.rs @@ -4,6 +4,10 @@ //! On save, we extract accounts (without bytecode) and storage from BlockchainDb //! and write a compact binary file. On load, we populate BlockchainDb directly, //! then seed bytecodes from the separate bytecodes.bin cache. +//! +//! The file format is raw bincode with no version header or magic bytes, so it +//! is not migratable: a cache written by a build with a different struct layout +//! decodes as a failure (cache miss) rather than being upgraded in place. use std::path::Path; use std::time::Instant; @@ -34,7 +38,19 @@ struct BinaryAccountInfo { /// Save the current BlockchainDb state to a binary file. /// /// This extracts accounts (without code) and storage from the MemDb -/// and serializes them with bincode for fast restoration. +/// and serializes them with bincode for fast restoration. Bytecode is excluded +/// and persisted separately to `bytecodes.bin`; the saved account info keeps +/// only the `code_hash`. +/// +/// Errors are logged at `warn` level and otherwise swallowed: serialization +/// failures, parent-directory creation failures, and write failures all return +/// without signalling to the caller, so a failed save is indistinguishable from +/// a successful one at the call site. +/// +/// The on-disk format is raw bincode with no version header, so it is not +/// forward/backward compatible: a file written by a build with a different +/// layout will fail to decode on load (treated as a cache miss) rather than +/// being migrated. pub fn save_binary_state(blockchain_db: &BlockchainDb, path: &Path) { let start = Instant::now(); @@ -91,6 +107,11 @@ pub fn save_binary_state(blockchain_db: &BlockchainDb, path: &Path) { /// Returns `true` if the binary state was loaded successfully, `false` otherwise. /// When successful, accounts (without code) and storage are populated in the MemDb. /// Bytecodes should be seeded separately from bytecodes.bin. +/// +/// Returns `false` (rather than erroring) when `path` cannot be read or its +/// contents fail to decode as the expected bincode layout; a decode failure is +/// logged at `warn` level. Because the format carries no version header, a file +/// written by an incompatible build is reported as a decode failure here. pub fn load_binary_state(blockchain_db: &BlockchainDb, path: &Path) -> bool { let start = Instant::now(); diff --git a/src/cache/bytecode.rs b/src/cache/bytecode.rs index 1a17271..a894a58 100644 --- a/src/cache/bytecode.rs +++ b/src/cache/bytecode.rs @@ -1,3 +1,16 @@ +//! On-disk cache of contract bytecode, keyed by account address. +//! +//! Bytecode is large and immutable for a deployed contract, so it is persisted +//! in its own file (`bytecodes.bin`) separately from the binary EVM state. On +//! save we copy the bytecode of every account that has any, and on load these +//! entries are used to re-seed the `code` of accounts that were restored +//! without it. +//! +//! Each entry's bytes are hex-encoded for the serde representation, but the +//! file is written as raw bincode with no version header, so a cache written by +//! an incompatible build fails to decode (cache miss) rather than being +//! migrated. + use std::collections::HashMap; use std::path::Path; @@ -24,6 +37,11 @@ pub(crate) struct BytecodeCache { impl BytecodeCache { /// Load bytecode cache from disk (binary format). + /// + /// Returns `None` if `path` cannot be read or its contents fail to decode as + /// bincode for this type; a decode failure is logged at `warn` level. The + /// format carries no version header, so a file from an incompatible build is + /// reported as `None`. pub(crate) fn load(path: &Path) -> Option { let data = std::fs::read(path).ok()?; bincode::deserialize(&data) @@ -32,6 +50,14 @@ impl BytecodeCache { } /// Save bytecode cache to disk (binary format). + /// + /// Creates the parent directory if needed, then writes the + /// bincode-serialized cache to `path`. + /// + /// # Errors + /// + /// Returns an error if the parent directory cannot be created, if bincode + /// serialization fails, or if writing the file fails. pub(crate) fn save(&self, path: &Path) -> Result<()> { if let Some(parent) = path.parent() { std::fs::create_dir_all(parent)?; @@ -42,6 +68,10 @@ impl BytecodeCache { } /// Merge new bytecodes from a BlockchainDb. + /// + /// Inserts (or overwrites) an entry for every account that currently has + /// non-empty `code`; accounts without loaded code are skipped. Existing + /// entries for addresses not present in `db` are left untouched. pub(crate) fn merge_from_db(&mut self, db: &BlockchainDb) { let accounts = db.accounts().read(); for (addr, info) in accounts.iter() { @@ -60,6 +90,86 @@ impl BytecodeCache { } } +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::{Bytes, U256}; + use foundry_fork_db::cache::BlockchainDbMeta; + use revm::state::{AccountInfo, Bytecode}; + + fn temp_path(tag: &str) -> std::path::PathBuf { + let dir = std::env::temp_dir().join(format!("evm_fork_cache_bytecode_{tag}")); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("create temp dir"); + dir.join("bytecodes.bin") + } + + #[test] + fn save_load_round_trip_through_hex_serde() { + let path = temp_path("roundtrip"); + let addr = Address::repeat_byte(0x42); + + let mut cache = BytecodeCache::default(); + cache.contracts.insert( + addr, + BytecodeCacheEntry { + bytecode: vec![0x60, 0x00, 0x60, 0x00, 0xf3], + }, + ); + cache.save(&path).expect("save bytecode cache"); + + let loaded = BytecodeCache::load(&path).expect("load bytecode cache"); + assert_eq!( + loaded.contracts.get(&addr).map(|e| e.bytecode.clone()), + Some(vec![0x60, 0x00, 0x60, 0x00, 0xf3]), + "bytecode survives the hex-encoded round trip" + ); + + let _ = std::fs::remove_dir_all(path.parent().unwrap()); + } + + #[test] + fn load_missing_file_is_none() { + assert!(BytecodeCache::load(std::path::Path::new("/nonexistent/bytecodes.bin")).is_none()); + } + + #[test] + fn merge_from_db_caches_only_coded_accounts() { + let db = BlockchainDb::new(BlockchainDbMeta::default(), None); + let coded = Address::repeat_byte(0x01); + let eoa = Address::repeat_byte(0x02); + + let code = Bytecode::new_raw(Bytes::from_static(&[0x60, 0x01, 0x60, 0x02, 0x01])); + let expected = code.original_byte_slice().to_vec(); + let code_hash = code.hash_slow(); + { + let mut accounts = db.accounts().write(); + accounts.insert( + coded, + AccountInfo { + balance: U256::ZERO, + nonce: 1, + code: Some(code), + code_hash, + account_id: None, + }, + ); + // An account with no loaded code must be skipped. + accounts.insert(eoa, AccountInfo::default()); + } + + let mut cache = BytecodeCache::default(); + cache.merge_from_db(&db); + + assert_eq!(cache.contracts.len(), 1, "only the coded account is cached"); + assert_eq!( + cache.contracts.get(&coded).map(|e| e.bytecode.clone()), + Some(expected) + ); + assert!(!cache.contracts.contains_key(&eoa)); + } +} + /// Hex serialization for bytecode bytes. mod hex_bytes { use alloy_primitives::hex; diff --git a/src/cache/metadata.rs b/src/cache/metadata.rs index 99538ce..6ab9472 100644 --- a/src/cache/metadata.rs +++ b/src/cache/metadata.rs @@ -17,6 +17,17 @@ use tracing::warn; use std::collections::HashSet; /// Configuration for disk-based caching of EVM state. +/// +/// Enables on-disk persistence of fetched fork state. Cache files are laid out +/// per chain under `cache_dir` (see [`CacheConfig::binary_state_cache_path`] and +/// the other path helpers), so multiple chains can share one base directory +/// without colliding. +/// +/// The `maintain_*` fields drive selective retention when state is reloaded: +/// `maintain_addresses` whitelists accounts whose storage is kept in full, while +/// `maintain_slots` whitelists individual slots for accounts whose remaining +/// storage should be purged. Together they let a cache load keep only the +/// long-lived state worth reusing and drop the rest. #[derive(Debug, Clone)] pub struct CacheConfig { /// Base directory for cache files. @@ -31,6 +42,10 @@ pub struct CacheConfig { impl CacheConfig { /// Create a new cache configuration. + /// + /// `cache_dir` is the base directory for all per-chain cache files, + /// `chain_id` namespaces them, and `maintain_addresses` / `maintain_slots` + /// select which state survives a reload (see the type-level docs). pub fn new( cache_dir: impl Into, chain_id: u64, @@ -76,6 +91,10 @@ impl CacheConfig { } /// Cached metadata for a UniswapV2 pool. +/// +/// Holds the immutable token pair plus a freshness marker +/// ([`last_block_timestamp`](Self::last_block_timestamp)) used to detect when +/// cached reserves have gone stale. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct V2PoolMetadata { pub token0: Address, @@ -88,6 +107,9 @@ pub struct V2PoolMetadata { } /// Cached metadata for a UniswapV3 pool. +/// +/// All fields are immutable for the lifetime of the pool: the token pair, the +/// fee tier, and the tick spacing. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct V3PoolMetadata { pub token0: Address, @@ -97,6 +119,10 @@ pub struct V3PoolMetadata { } /// Cached metadata for a Balancer pool. +/// +/// Holds the pool's tokens, weights, and swap fee plus a freshness marker +/// ([`last_change_block`](Self::last_change_block)) used to detect when cached +/// balances have gone stale. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct BalancerPoolMetadata { pub tokens: Vec
, @@ -131,6 +157,14 @@ pub struct ImmutableDataCache { impl ImmutableDataCache { /// Load immutable data cache from disk (binary format). + /// + /// Returns `None` if `path` cannot be read or the contents are not valid + /// bincode for this type (a parse failure is logged at `warn` level and + /// swallowed). Callers should treat `None` as "no cache yet" and start fresh. + /// + /// Note: the on-disk format is bincode with no version header, so a cache + /// written by an incompatible build deserializes as a parse failure (`None`) + /// rather than being migrated. pub fn load(path: &Path) -> Option { let data = std::fs::read(path).ok()?; bincode::deserialize(&data) @@ -139,6 +173,14 @@ impl ImmutableDataCache { } /// Save immutable data cache to disk (binary format). + /// + /// Creates the parent directory if it does not exist, then writes the + /// bincode-serialized cache to `path`. + /// + /// # Errors + /// + /// Returns an error if the parent directory cannot be created, if bincode + /// serialization fails, or if writing the file fails. pub fn save(&self, path: &Path) -> Result<()> { if let Some(parent) = path.parent() { std::fs::create_dir_all(parent)?; @@ -179,17 +221,26 @@ impl ImmutableDataCache { } /// Get cached Balancer pool metadata. + /// + /// The `pool_id` is keyed by its `Debug` formatting (matching + /// [`ImmutableDataCache::set_balancer_pool`]), so a lookup only hits if the + /// id was stored through that same setter. pub fn get_balancer_pool(&self, pool_id: B256) -> Option<&BalancerPoolMetadata> { self.balancer_pools.get(&format!("{:?}", pool_id)) } /// Cache Balancer pool metadata. + /// + /// The `pool_id` is stored under its `Debug` formatting as the map key. pub fn set_balancer_pool(&mut self, pool_id: B256, metadata: BalancerPoolMetadata) { self.balancer_pools .insert(format!("{:?}", pool_id), metadata); } /// Check if the cache is empty. + /// + /// Returns `true` only when every sub-map (token decimals and all pool + /// kinds) is empty. pub fn is_empty(&self) -> bool { self.token_decimals.is_empty() && self.v2_pools.is_empty() @@ -198,6 +249,9 @@ impl ImmutableDataCache { } /// Get the total number of cached entries. + /// + /// This is the sum of the entry counts across all sub-maps (token decimals + /// plus V2, V3, and Balancer pools), not a count of distinct addresses. pub fn len(&self) -> usize { self.token_decimals.len() + self.v2_pools.len() diff --git a/src/cache/mod.rs b/src/cache/mod.rs index aefb749..b83a640 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -17,6 +17,7 @@ pub use overlay::EvmOverlay; pub use slot_observations::SlotObservationTracker; pub use snapshot::EvmSnapshot; #[cfg(feature = "protocols")] +#[cfg_attr(docsrs, doc(cfg(feature = "protocols")))] pub use storage_keys::{ PANCAKE_V3_LIQUIDITY_SLOT, PANCAKE_V3_TICK_BITMAP_BASE_SLOT, PANCAKE_V3_TICKS_BASE_SLOT, SLIPSTREAM_LIQUIDITY_SLOT, SLIPSTREAM_SLOT0_SLOT, SLIPSTREAM_TICK_BITMAP_BASE_SLOT, @@ -26,6 +27,7 @@ pub use storage_keys::{ v3_tick_info_storage_keys_with_base, }; #[cfg(feature = "protocols")] +#[cfg_attr(docsrs, doc(cfg(feature = "protocols")))] pub use tick_snapshot::{SerializableTickInfo, TickInfo, V3PoolTickSnapshot, V3TickSnapshotCache}; use std::{ @@ -86,8 +88,18 @@ pub type RpcCallFn = Arc Result + Send + Sync>; /// Used by V3 tick prefetch to avoid 16K+ individual channel round-trips through /// SharedBackend. Fires concurrent `eth_getStorageAt` calls directly via the provider /// and returns results for bulk injection into BlockchainDb. -pub type StorageBatchFetchFn = - Arc) -> Vec<(Address, U256, Result)> + Send + Sync>; +/// +/// The second argument pins the fetch to a specific block: `Some(block)` fetches +/// at exactly that block, while `None` uses the fetcher's configured block (the +/// cache's currently-pinned block). The freshness validator passes the block its +/// snapshot was built from, so a concurrent [`EvmCache::set_block`] cannot make +/// the deferred fetch read a *different* block than the snapshot it is compared +/// against. +pub type StorageBatchFetchFn = Arc< + dyn Fn(Vec<(Address, U256)>, Option) -> Vec<(Address, U256, Result)> + + Send + + Sync, +>; /// Return a tokio runtime [`Handle`] suitable for `block_in_place` + `block_on`, /// or an error describing why one is unavailable. @@ -119,12 +131,22 @@ fn block_in_place_handle() -> Result { static CACHE_SPEED_MODE: AtomicU8 = AtomicU8::new(CacheSpeedMode::Slow as u8); /// Runtime tuning profile for cache-side batch storage fetches. +/// +/// Selects the per-batch size and concurrency used by [`StorageBatchFetchFn`]: +/// faster modes send larger batches with more in-flight HTTP requests, slower +/// modes throttle to avoid RPC rate-limiting (e.g. HTTP 429 on Base). The +/// selected mode is **process-global** state, set via [`set_cache_speed_mode`] +/// and read via [`cache_speed_mode`]; it affects every cache in the process. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[repr(u8)] pub enum CacheSpeedMode { + /// Largest batches, highest concurrency — fastest, most likely to trip rate limits. Fast = 0, + /// Moderate batch size and concurrency. Normal = 1, + /// Conservative batch size and concurrency. The default. Slow = 2, + /// Smallest batches, single in-flight request — slowest, gentlest on the RPC provider. XSlow = 3, } @@ -140,12 +162,20 @@ impl CacheSpeedMode { } } -/// Set the global cache batch-fetch speed profile. +/// Set the process-global cache batch-fetch speed profile. +/// +/// This mutates a single static shared by every cache in the process, so it +/// affects all in-flight and future batch fetches, not just one [`EvmCache`]. +/// Read the current value with [`cache_speed_mode`]. pub fn set_cache_speed_mode(mode: CacheSpeedMode) { CACHE_SPEED_MODE.store(mode as u8, Ordering::Relaxed); } -/// Return the current global cache batch-fetch speed profile. +/// Return the current process-global cache batch-fetch speed profile. +/// +/// Defaults to [`CacheSpeedMode::Slow`] until changed via +/// [`set_cache_speed_mode`]. The value is shared across all caches in the +/// process. pub fn cache_speed_mode() -> CacheSpeedMode { CacheSpeedMode::from_u8(CACHE_SPEED_MODE.load(Ordering::Relaxed)) } @@ -168,15 +198,22 @@ pub enum MissingTargetBehavior { /// simulator, so a non-zero `value` does not require the caller to be funded. #[derive(Debug, Clone, Default)] pub struct TxConfig { - /// Native value (wei) sent with the call. + /// Native value (wei) sent with the call. Set this to simulate a payable + /// function or a native-ETH transfer. Balance checks are disabled in the + /// simulator, so the caller need not be funded for a non-zero value. pub value: U256, - /// Gas limit; `None` uses revm's default. + /// Gas limit for the call. `None` uses revm's default. Set this to model a + /// gas-bounded call (e.g. to observe out-of-gas behavior). pub gas_limit: Option, - /// Gas price (wei); `None` uses revm's default. + /// Gas price (wei) for the call. `None` uses revm's default. Rarely needed + /// because base-fee checks are disabled in the simulator. pub gas_price: Option, - /// Sender nonce; `None` lets the simulator pick (nonce checks are disabled). + /// Sender nonce. `None` lets the simulator pick; nonce checks are disabled, + /// so this is only worth setting when a contract reads the nonce explicitly. pub nonce: Option, - /// EIP-2930 access list to pre-warm slots for this call. + /// EIP-2930 access list to pre-warm accounts and storage slots for this + /// call. Pre-warming changes EIP-2929 gas accounting; supply it when + /// reproducing the gas cost of a transaction that carried an access list. pub access_list: Option, } @@ -225,12 +262,20 @@ where } /// Pin simulations and RPC fetches to a specific block. + /// + /// Use this to fork at a fixed height for reproducible simulation. Without + /// a call to [`block`](Self::block) or [`latest_block`](Self::latest_block) + /// the builder defaults to the latest block at [`build`](Self::build) time. pub fn block(mut self, block: BlockId) -> Self { self.block = Some(block); self } /// Pin to the latest block. + /// + /// The height is resolved when [`build`](Self::build) fetches the block + /// header, so the cache forks at whatever was latest at construction. Use + /// [`block`](Self::block) instead to pin a fixed, reproducible height. pub fn latest_block(mut self) -> Self { self.block = Some(BlockId::latest()); self @@ -243,6 +288,12 @@ where } /// Enable disk-backed caching with the given configuration. + /// + /// Supplying a [`CacheConfig`] turns on persistence of EVM state, + /// bytecodes, immutable data, and (with the `protocols` feature) V3 tick + /// snapshots under the configured chain directory; the cache is loaded on + /// [`build`](Self::build) and flushed on drop. Omit it for a purely + /// in-memory cache backed solely by RPC. pub fn cache_config(mut self, cache_config: CacheConfig) -> Self { self.cache_config = Some(cache_config); self @@ -328,10 +379,45 @@ pub struct EvmCache { spec_id: SpecId, } +/// Outcome of a balance-delta-tracking simulation. +/// +/// Produced by [`EvmCache::simulate_call_with_balance_deltas`] and +/// [`EvmCache::simulate_with_transfer_tracking`]: a successful call together +/// with the per-token balance changes it caused, its emitted logs, the touched +/// access list, and its raw return data. +/// Execution outcome of a simulated call. +/// +/// Lets a caller distinguish a successful call — even one that emitted no logs, +/// such as a view call — from a revert or a halt, without guessing from `logs` +/// or `output`. Revert payloads live in [`CallSimulationResult::output`] and can +/// be decoded with [`RevertDecoder`](crate::errors::RevertDecoder); only `Halt` +/// carries extra data here, since its reason has nowhere else to live. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SimStatus { + /// The call returned successfully. + Success, + /// The call reverted; the revert payload (if any) is in `output`. + Revert, + /// The call halted (e.g. out of gas, invalid opcode). + Halt { + /// Debug-formatted halt reason. + reason: String, + }, +} + #[derive(Clone, Debug)] +#[non_exhaustive] pub struct CallSimulationResult { + /// Whether the call succeeded, reverted, or halted. + pub status: SimStatus, + /// Gas consumed by the (successful) call. pub gas_used: u64, + /// Net change in `owner`'s balance per tracked token, as a **signed** + /// [`I256`] (`post - pre`): positive means the call increased the balance, + /// negative means it decreased it. Tokens not seen by the call may be + /// absent or zero. pub token_deltas: HashMap, + /// Logs emitted by the call (in emission order). pub logs: Vec, /// EIP-2930 access list of all accounts and storage slots touched during simulation. /// Extracted from the EVM journaled state after execution. @@ -610,8 +696,8 @@ impl EvmCache { let provider_for_batch = provider.clone(); let batch_block_id = Arc::new(Mutex::new(block_id)); let batch_block_ref = batch_block_id.clone(); - let storage_batch_fetcher: StorageBatchFetchFn = - Arc::new(move |requests: Vec<(Address, U256)>| { + let storage_batch_fetcher: StorageBatchFetchFn = Arc::new( + move |requests: Vec<(Address, U256)>, block: Option| { use futures::stream::{self, StreamExt}; // Max items per JSON-RPC batch. RPC providers typically limit batch // size to ~1000 items. Reduced from 200 to avoid 429s on Base. @@ -644,7 +730,11 @@ impl EvmCache { .collect(); } }; - let current_block = *batch_block_ref.lock().unwrap(); + // Pin to the explicitly-requested block when given, else the + // cache's currently-pinned block. Capturing the block at the call + // site is what lets the deferred freshness validator fetch at the + // snapshot's block despite a later `set_block`. + let current_block = block.unwrap_or_else(|| *batch_block_ref.lock().unwrap()); tokio::task::block_in_place(|| { handle.block_on(async { let mut results = Vec::with_capacity(requests.len()); @@ -727,7 +817,8 @@ impl EvmCache { results }) }) - }); + }, + ); // Spawn the backend handler on a background task let backend = @@ -903,21 +994,42 @@ impl EvmCache { } /// Get the cache configuration, if any. + /// + /// Returns `None` when the cache is purely in-memory (no disk persistence), + /// i.e. constructed without a [`CacheConfig`] or via + /// [`from_backend`](Self::from_backend). pub fn cache_config(&self) -> Option<&CacheConfig> { self.cache_config.as_ref() } - /// Get a reference to the underlying BlockchainDb. + /// Get a reference to the underlying [`BlockchainDb`] (the layer-2 backend + /// store of accounts, storage, and bytecodes). + /// + /// This exposes an internal store and bypasses the cache's two-layer + /// consistency model: reads here see only the backend layer, not the + /// CacheDB overlay, and any writes performed through it skip the overlay. + /// Prefer the higher-level accessors; use with care. pub fn blockchain_db(&self) -> &BlockchainDb { &self.blockchain_db } - /// Get a reference to the underlying SharedBackend. + /// Get a reference to the underlying [`SharedBackend`] (the lazy RPC-backed + /// fetcher shared across clones). + /// + /// This exposes an internal and bypasses the cache's two-layer consistency + /// model: it reads/fetches directly without consulting the CacheDB overlay. + /// Prefer the higher-level accessors; use with care. pub fn backend(&self) -> &SharedBackend { &self.backend } - /// Get a mutable reference to the database. + /// Get a mutable reference to the underlying [`ForkCacheDB`] (the layer-1 + /// CacheDB overlay). + /// + /// This exposes an internal and bypasses the cache's two-layer consistency + /// model: writes made here land only in the overlay and are not mirrored + /// into the BlockchainDb backend, so parallel tasks sharing the backend + /// will not see them. Prefer the higher-level mutators; use with care. pub fn db_mut(&mut self) -> &mut ForkCacheDB { &mut self.db } @@ -927,6 +1039,14 @@ impl EvmCache { /// This is much faster than `call_raw` for batch operations because the RPC /// node has all state in memory and doesn't need lazy storage fetching. /// Returns `None` if no RPC caller is available (e.g. `from_backend` constructor). + /// + /// # Panics + /// Must be called from within a **multi-thread** tokio runtime: the callback + /// drives the async `eth_call` to completion via + /// `tokio::task::block_in_place`. On a current-thread runtime (or with no + /// runtime), the callback degrades to an `Err` rather than panicking, but + /// `block_in_place` itself will panic if invoked from a non-worker thread of + /// a multi-thread runtime. pub fn rpc_call(&self, to: Address, calldata: Bytes) -> Option> { self.rpc_caller .as_ref() @@ -936,6 +1056,15 @@ impl EvmCache { /// Get the batch storage fetcher, if available. /// /// Returns `None` when constructed via `from_backend` (no provider available). + /// + /// # Panics + /// The returned [`StorageBatchFetchFn`] must be invoked from within a + /// **multi-thread** tokio runtime: it drives concurrent `eth_getStorageAt` + /// calls to completion via `tokio::task::block_in_place`. On a + /// current-thread runtime (or with no runtime) it degrades to an `Err` + /// result for every requested slot rather than panicking, but + /// `block_in_place` itself will panic if invoked from a non-worker thread of + /// a multi-thread runtime. pub fn storage_batch_fetcher(&self) -> Option<&StorageBatchFetchFn> { self.storage_batch_fetcher.as_ref() } @@ -952,6 +1081,40 @@ impl EvmCache { } } + /// Inject freshly-fetched storage values, healing **both** cache layers. + /// + /// Like [`inject_storage_batch`](Self::inject_storage_batch) this writes each + /// value into the BlockchainDb backend (layer 2). Additionally, for any + /// address that *already* has a CacheDB overlay entry (layer 1), it writes + /// the slot into that overlay too. + /// + /// This matters because both [`create_snapshot`](Self::create_snapshot) and + /// the synchronous EVM SLOAD path let the overlay win over the backend. A + /// correction written only to layer 2 would be shadowed by a stale layer-1 + /// slot, so the cache could never converge — the freshness validator would + /// re-detect the same change and re-correct it every cycle. Writing through + /// the overlay keeps the layer that wins authoritative. + /// + /// It deliberately does **not** create a new overlay account for an address + /// that has none: such a slot is layer-2-only (e.g. cold prefetch), where + /// the backend write is already authoritative and materializing an overlay + /// entry would pollute layer 1 and could shadow later RPC reads. + pub fn inject_storage_batch_fresh(&mut self, results: &[(Address, U256, U256)]) { + { + let mut storage = self.blockchain_db.storage().write(); + for &(addr, slot, value) in results { + storage.entry(addr).or_default().insert(slot, value); + } + } + // Write through to the overlay only for accounts already materialized + // there, so the winning layer reflects the fresh value. + for &(addr, slot, value) in results { + if let Some(db_account) = self.db.cache.accounts.get_mut(&addr) { + db_account.storage.insert(slot, value); + } + } + } + /// Set (or replace) the batch storage fetcher. /// /// This is the seam the freshness controller and tests use to drive @@ -1006,7 +1169,7 @@ impl EvmCache { .map(|&(addr, slot)| ((addr, slot), self.cached_storage_value(addr, slot))) .collect(); - let results = (fetcher)(slots.to_vec()); + let results = (fetcher)(slots.to_vec(), self.block); let mut changed = Vec::new(); let mut to_inject = Vec::new(); @@ -1037,7 +1200,7 @@ impl EvmCache { } if !to_inject.is_empty() { - self.inject_storage_batch(&to_inject); + self.inject_storage_batch_fresh(&to_inject); } Ok(changed) } @@ -1075,20 +1238,35 @@ impl EvmCache { } } - /// Get the chain ID used for EVM simulations. + /// Get the chain ID used for EVM simulations (the `CHAINID` opcode). pub fn chain_id(&self) -> u64 { self.chain_id } - /// Create a snapshot of the current cache state for later restoration. + /// Take a low-level, same-thread snapshot of the CacheDB overlay for + /// in-place restore. /// - /// Note: This creates a copy of the inner cache only (accounts and storage), - /// not the underlying database wrapper. + /// Clones the inner [`revm::database::Cache`] (the layer-1 overlay's + /// accounts and storage) only — not the underlying database wrapper or the + /// BlockchainDb backend. Pair with [`restore`](Self::restore) to roll the + /// overlay back on the same `EvmCache` after speculative mutations (this is + /// how the balance-slot scan probes and rewinds). + /// + /// For cross-thread fan-out use [`create_snapshot`](Self::create_snapshot) + /// instead: it merges both layers into an `Arc<`[`EvmSnapshot`]`>` that is + /// `Send + Sync` and can be shared with parallel simulators via + /// [`EvmOverlay`]. pub fn snapshot(&self) -> revm::database::Cache { self.db.cache.clone() } - /// Restore the cache state from a previous snapshot. + /// Restore the CacheDB overlay from a snapshot taken with + /// [`snapshot`](Self::snapshot). + /// + /// Overwrites the layer-1 overlay wholesale with `snapshot`, discarding any + /// overlay mutations made since it was taken. The BlockchainDb backend is + /// untouched. This is the in-place counterpart to the cross-thread + /// [`create_snapshot`](Self::create_snapshot) / [`EvmOverlay`] path. pub fn restore(&mut self, snapshot: revm::database::Cache) { self.db.cache = snapshot; } @@ -1104,7 +1282,8 @@ impl EvmCache { } } - /// Create an immutable snapshot of the current EVM state. + /// Create an immutable snapshot of the current EVM state for cross-thread + /// fan-out. /// /// Merges both layers (CacheDB overlay + BlockchainDb backend) into a /// single flat HashMap. The snapshot is `Send + Sync` and can be shared @@ -1112,6 +1291,9 @@ impl EvmCache { /// /// CacheDB overlay values take precedence over BlockchainDb values. /// Use with [`EvmOverlay`] for parallel simulation. + /// + /// For cheap same-thread save/restore of just the overlay, prefer + /// [`snapshot`](Self::snapshot) / [`restore`](Self::restore) instead. pub fn create_snapshot(&self) -> Arc { let mut accounts = HashMap::new(); let mut storage = HashMap::new(); @@ -1199,7 +1381,11 @@ impl EvmCache { } } - /// Get the current block. + /// Get the block that RPC fetches are currently pinned to. + /// + /// `None` means no explicit pin was set, so the backend reads the latest + /// block. Set it with [`set_block`](Self::set_block) or + /// [`repin_to_block`](Self::repin_to_block). pub fn block(&self) -> Option { self.block } @@ -1222,12 +1408,25 @@ impl EvmCache { self.timestamp_override } - /// Get the block number used for EVM simulations (NUMBER opcode). + /// Get the block number used for EVM simulations (the `NUMBER` opcode). + /// + /// Fetched from the pinned block's header at construction and kept in + /// lockstep with the pin by [`set_block`](Self::set_block) / + /// [`repin_to_block`](Self::repin_to_block). `None` means revm falls back + /// to `0`, which can steer contracts that branch on `block.number` down a + /// different code path. Override directly via + /// [`set_block_context`](Self::set_block_context). pub fn block_number(&self) -> Option { self.block_number } - /// Get the base fee used for EVM simulations (BASEFEE opcode). + /// Get the base fee per gas used for EVM simulations (the `BASEFEE` opcode). + /// + /// Fetched from the pinned block's header at construction. `None` means + /// revm falls back to `0`. Unlike `block_number` this is **not** refreshed + /// by [`set_block`](Self::set_block); refresh it with + /// [`set_block_context`](Self::set_block_context) after fetching a new + /// header if `BASEFEE` accuracy matters. pub fn basefee(&self) -> Option { self.basefee } @@ -1241,17 +1440,29 @@ impl EvmCache { self.basefee = basefee; } - /// Override the block beneficiary (COINBASE opcode) for subsequent simulations. + /// Override the block beneficiary (the `COINBASE` opcode) for subsequent + /// simulations. + /// + /// Set this when simulating logic that reads `block.coinbase` (e.g. + /// MEV/builder tip accounting). `None` lets revm use its default beneficiary. pub fn set_coinbase(&mut self, coinbase: Option
) { self.coinbase = coinbase; } - /// Override `prevrandao` (PREVRANDAO opcode) for subsequent simulations. + /// Override `prevrandao` (the `PREVRANDAO` opcode, the post-merge header mix + /// hash) for subsequent simulations. + /// + /// Set this when reproducing contracts that source on-chain randomness from + /// `block.prevrandao`. `None` leaves revm's default in place. pub fn set_prevrandao(&mut self, prevrandao: Option) { self.prevrandao = prevrandao; } - /// Override the block gas limit (GASLIMIT opcode) for subsequent simulations. + /// Override the block gas limit (the `GASLIMIT` opcode) for subsequent + /// simulations. + /// + /// Set this when simulating logic that reads `block.gaslimit`. `None` lets + /// revm use its default. pub fn set_block_gas_limit(&mut self, gas_limit: Option) { self.block_gas_limit = gas_limit; } @@ -1333,14 +1544,32 @@ impl EvmCache { Ok(()) } - /// Pre-seed known ERC20 balance mapping slots so that `set_erc20_balance_with_slot_scan` - /// can skip the scanning step for these tokens. + /// Pre-seed known ERC20 `balanceOf` mapping base slots, keyed by token. + /// + /// Each `(token, slot)` records the storage slot of the token's + /// `mapping(address => uint256) balances`, letting + /// [`set_erc20_balance_with_slot_scan`](Self::set_erc20_balance_with_slot_scan) + /// skip its `0..=max_slot` probing pass for that token and write the balance + /// directly. Seeding a wrong slot is self-correcting: the scan verifies the + /// write and falls back to a fresh probe (evicting the bad seed) if it + /// fails. Later entries overwrite earlier ones for the same token. pub fn seed_erc20_balance_slots(&mut self, slots: impl IntoIterator) { for (token, slot) in slots { self.erc20_balance_slots.insert(token, slot); } } + /// Write a value into a Solidity `mapping(address => ...)` entry on + /// `contract`, at the mapping declared at base slot `slot`. + /// + /// Computes the entry's storage key as + /// `keccak256(abi.encode(slot_address, slot))` — Solidity's layout for an + /// address-keyed mapping — and writes `value` there in the CacheDB overlay. + /// Used to forge ERC20 balances and allowances without an on-chain transfer. + /// + /// # Errors + /// Returns an error if the underlying CacheDB storage insert fails (e.g. the + /// account cannot be loaded from the backend). pub fn insert_mapping_storage_slot( &mut self, contract: Address, @@ -1441,6 +1670,7 @@ impl EvmCache { /// * `pool_address` - The UniswapV2 pair contract address /// * `metadata` - The cached pool metadata containing token0 and token1 #[cfg(feature = "protocols")] + #[cfg_attr(docsrs, doc(cfg(feature = "protocols")))] pub fn inject_v2_pool_metadata( &mut self, pool_address: Address, @@ -1474,6 +1704,7 @@ impl EvmCache { /// * `pool_address` - The UniswapV3 pool contract address /// * `tick_bitmap` - Map of word position (int16) to bitmap value (uint256) #[cfg(feature = "protocols")] + #[cfg_attr(docsrs, doc(cfg(feature = "protocols")))] pub fn inject_v3_tick_bitmap( &mut self, pool_address: Address, @@ -1486,6 +1717,7 @@ impl EvmCache { /// /// PancakeSwap V3 uses base slot 7 instead of Uniswap V3's slot 6. #[cfg(feature = "protocols")] + #[cfg_attr(docsrs, doc(cfg(feature = "protocols")))] pub fn inject_v3_tick_bitmap_with_base( &mut self, pool_address: Address, @@ -1531,6 +1763,7 @@ impl EvmCache { /// * `pool_address` - The UniswapV3 pool contract address /// * `ticks` - Map of tick index (int24) to tick info #[cfg(feature = "protocols")] + #[cfg_attr(docsrs, doc(cfg(feature = "protocols")))] pub fn inject_v3_ticks( &mut self, pool_address: Address, @@ -1543,6 +1776,7 @@ impl EvmCache { /// /// PancakeSwap V3 uses ticks at slot 6 instead of Uniswap V3's slot 5. #[cfg(feature = "protocols")] + #[cfg_attr(docsrs, doc(cfg(feature = "protocols")))] pub fn inject_v3_ticks_with_base( &mut self, pool_address: Address, @@ -1684,6 +1918,15 @@ impl EvmCache { Ok((result, access_list)) } + /// Execute a call and return its emitted logs and gas used. + /// + /// A thin wrapper over [`call`](Self::call) that requires success and + /// discards the return data. When `commit` is true the call's state changes + /// are persisted to the CacheDB overlay; otherwise they are reverted. + /// + /// # Errors + /// Returns an error if the underlying transact fails, or if the call did not + /// `Success` (i.e. it reverted or halted). pub fn call_logs( &mut self, from: Address, @@ -1699,6 +1942,14 @@ impl EvmCache { } } + /// Read an ERC20 token balance by simulating a `balanceOf(owner)` call. + /// + /// Non-committing: the read is reverted, so it never mutates cache state. + /// + /// # Errors + /// Returns an error if the simulated call fails or does not `Success` (e.g. + /// `token` is not a contract or reverts), or if the returned data cannot be + /// ABI-decoded as a `uint256`. pub fn erc20_balance_of(&mut self, token: Address, owner: Address) -> Result { let call = IERC20::balanceOfCall { target: owner }; let result = self.call_raw(Address::ZERO, token, Bytes::from(call.abi_encode()), false)?; @@ -1714,6 +1965,14 @@ impl EvmCache { } } + /// Read an ERC20 allowance by simulating an `allowance(owner, spender)` call. + /// + /// Non-committing: the read is reverted, so it never mutates cache state. + /// + /// # Errors + /// Returns an error if the simulated call fails or does not `Success` (e.g. + /// `token` is not a contract or reverts), or if the returned data cannot be + /// ABI-decoded as a `uint256`. pub fn erc20_allowance( &mut self, token: Address, @@ -1734,6 +1993,21 @@ impl EvmCache { } } + /// Read an ERC20 token's decimals by simulating a `decimals()` call. + /// + /// Memoized: a hit in the in-memory token-decimals map returns immediately + /// without simulating. On a miss the value is resolved by a non-committing + /// `decimals()` call. + /// + /// # Side effects + /// On a miss the resolved value is cached in **both** the in-memory + /// token-decimals map (process lifetime) **and** the immutable data cache + /// (so it is persisted to disk on the next [`flush`](Self::flush)). + /// + /// # Errors + /// Returns an error if the simulated call fails or does not `Success` (e.g. + /// `token` is not a contract or reverts), or if the returned data cannot be + /// ABI-decoded as a `uint8`. pub fn erc20_decimals(&mut self, token: Address) -> Result { if let Some(decimals) = self.token_decimals.get(&token) { return Ok(*decimals); @@ -1756,24 +2030,36 @@ impl EvmCache { } } - /// Get a reference to the immutable data cache. + /// Get a reference to the immutable data cache (token decimals and pool + /// metadata that never change for a given contract). pub fn immutable_cache(&self) -> &ImmutableDataCache { &self.immutable_cache } /// Get a mutable reference to the immutable data cache. + /// + /// Use this to pre-populate token decimals or pool metadata that would + /// otherwise be discovered lazily. Entries are persisted on the next + /// [`flush`](Self::flush) (and on drop) when a [`CacheConfig`] is set. pub fn immutable_cache_mut(&mut self) -> &mut ImmutableDataCache { &mut self.immutable_cache } - /// Get a reference to the V3 tick snapshot cache. + /// Get a reference to the V3 pool tick snapshot cache (per-pool + /// `tick_bitmap`, `ticks`, and liquidity used for liquidity validation). #[cfg(feature = "protocols")] + #[cfg_attr(docsrs, doc(cfg(feature = "protocols")))] pub fn tick_snapshot_cache(&self) -> &V3TickSnapshotCache { &self.tick_snapshot_cache } - /// Get a mutable reference to the V3 tick snapshot cache. + /// Get a mutable reference to the V3 pool tick snapshot cache. + /// + /// Use this to insert or update tick snapshots. Entries are persisted on + /// the next [`flush`](Self::flush) (and on drop) when a [`CacheConfig`] is + /// set. #[cfg(feature = "protocols")] + #[cfg_attr(docsrs, doc(cfg(feature = "protocols")))] pub fn tick_snapshot_cache_mut(&mut self) -> &mut V3TickSnapshotCache { &mut self.tick_snapshot_cache } @@ -2092,6 +2378,22 @@ impl EvmCache { .unwrap_or(0) } + /// Simulate a call and compute `owner`'s net balance change for each token + /// in `tokens` by reading `balanceOf(owner)` immediately before and after. + /// + /// Each delta is the signed `post - pre` difference (see + /// [`CallSimulationResult::token_deltas`]). When `commit` is true the call's + /// state changes are persisted to the CacheDB overlay; otherwise they are + /// reverted. Unlike + /// [`simulate_with_transfer_tracking`](Self::simulate_with_transfer_tracking), + /// this measures deltas via pre/post balance reads (not transfer-event + /// inspection) and the returned + /// [`access_list`](CallSimulationResult::access_list) is always empty. + /// + /// # Errors + /// Returns an error if building the tx env fails, if a pre/post + /// `balanceOf` read fails, or if the call does not `Success` (i.e. it + /// reverted or halted). On error the simulation is reverted. pub fn simulate_call_with_balance_deltas( &mut self, from: Address, @@ -2145,6 +2447,7 @@ impl EvmCache { evm.journaled_state.checkpoint_revert(checkpoint); } Ok(CallSimulationResult { + status: SimStatus::Success, gas_used, token_deltas, logs, @@ -2220,6 +2523,7 @@ impl EvmCache { } Ok(CallSimulationResult { + status: SimStatus::Success, gas_used, token_deltas, logs, @@ -2268,6 +2572,11 @@ impl EvmCache { /// /// Note: This commits the deployment to the CacheDB. Use a throw-away deployer /// address (e.g., `Address::ZERO`) to avoid side effects on real accounts. + /// + /// # Errors + /// Returns an error if the CREATE tx env cannot be built, if the deployment + /// reverts or halts, or if it succeeds but the EVM returns no contract + /// address. pub fn deploy_contract(&mut self, from: Address, creation_code: Bytes) -> Result
{ let tx = TxEnv::builder() .caller(from) @@ -2306,6 +2615,13 @@ impl EvmCache { /// at `target` remain unchanged. `target` must already have non-empty runtime /// bytecode. Both the CacheDB overlay and BlockchainDb backend are updated, /// ensuring the override is visible to parallel EVM tasks sharing the same backend. + /// + /// # Errors + /// Returns an error if `source` has no cached bytecode or its code is empty, + /// if `target` cannot be loaded (it must already exist on the backend), or + /// if `target` has no existing runtime bytecode to override. For synthetic + /// `target` addresses that may not exist, use + /// [`override_or_create_account_code`](Self::override_or_create_account_code). pub fn override_account_code(&mut self, source: Address, target: Address) -> Result<()> { self.override_account_code_with_missing_target(source, target, MissingTargetBehavior::Error) } @@ -2624,6 +2940,12 @@ impl<'a> EvmSession<'a> { } /// Get access to the underlying EVM for advanced operations. + /// + /// This exposes revm internals and bypasses the cache's two-layer + /// consistency model: state mutated directly through the journaled EVM + /// lands in the session's journal, not the BlockchainDb backend, and is + /// only flushed to the CacheDB overlay on [`commit`](Self::commit). Use + /// with care. pub fn evm(&mut self) -> &mut CacheEvm<'a> { &mut self.evm } @@ -2660,7 +2982,7 @@ fn extract_access_list(state: &revm::state::EvmState) -> AccessList { AccessList(items) } -#[cfg(test)] +#[cfg(all(test, feature = "protocols"))] mod tests { use super::*; use storage_keys::{i128_to_u256, i256_from_i16, i256_from_i24}; @@ -2965,65 +3287,6 @@ mod tests { assert_ne!(slot_60, slot_neg60); } - // ==================== V2 pool metadata injection tests ==================== - - #[test] - fn test_v2_pool_metadata_storage_slots() { - // Verify the storage slot constants match UniswapV2Pair layout - const TOKEN0_SLOT: U256 = U256::from_limbs([6, 0, 0, 0]); - const TOKEN1_SLOT: U256 = U256::from_limbs([7, 0, 0, 0]); - - // Slots should be sequential starting at 6 - assert_eq!(TOKEN0_SLOT, U256::from(6)); - assert_eq!(TOKEN1_SLOT, U256::from(7)); - } - - #[test] - fn test_address_to_u256_conversion() { - // Test that address conversion preserves the address bytes correctly - let addr = Address::repeat_byte(0xAB); - let value = U256::from_be_slice(addr.as_slice()); - - // Address is 20 bytes, should be right-aligned in U256 (32 bytes) - let bytes = value.to_be_bytes::<32>(); - - // First 12 bytes should be zero (padding) - assert_eq!(&bytes[..12], &[0u8; 12]); - - // Last 20 bytes should be the address - assert_eq!(&bytes[12..], addr.as_slice()); - } - - #[test] - fn test_v2_metadata_address_values() { - // Test specific address encoding - let token0 = Address::repeat_byte(0x11); - let token1 = Address::repeat_byte(0x22); - - let metadata = V2PoolMetadata { - token0, - token1, - last_block_timestamp: 0, - }; - - let token0_value = U256::from_be_slice(metadata.token0.as_slice()); - let token1_value = U256::from_be_slice(metadata.token1.as_slice()); - - // Values should be different - assert_ne!(token0_value, token1_value); - - // Each should be non-zero - assert_ne!(token0_value, U256::ZERO); - assert_ne!(token1_value, U256::ZERO); - - // Verify round-trip: extract address bytes back - let token0_bytes = token0_value.to_be_bytes::<32>(); - let token1_bytes = token1_value.to_be_bytes::<32>(); - - assert_eq!(&token0_bytes[12..], token0.as_slice()); - assert_eq!(&token1_bytes[12..], token1.as_slice()); - } - // -- PancakeSwap V3 storage slot tests -- #[test] @@ -3105,6 +3368,73 @@ mod tests { assert_eq!(keys[2], keys[0] + U256::from(2)); assert_eq!(keys[3], keys[0] + U256::from(3)); } +} + +/// Tests that exercise only the generic (protocol-independent) engine, so they +/// run under `--no-default-features` too. The protocol-gated unit tests live in +/// the `tests` module above, which is `#[cfg(feature = "protocols")]`. +#[cfg(test)] +mod core_tests { + use super::*; + + // ==================== V2 pool metadata injection tests ==================== + + #[test] + fn test_v2_pool_metadata_storage_slots() { + // Verify the storage slot constants match UniswapV2Pair layout + const TOKEN0_SLOT: U256 = U256::from_limbs([6, 0, 0, 0]); + const TOKEN1_SLOT: U256 = U256::from_limbs([7, 0, 0, 0]); + + // Slots should be sequential starting at 6 + assert_eq!(TOKEN0_SLOT, U256::from(6)); + assert_eq!(TOKEN1_SLOT, U256::from(7)); + } + + #[test] + fn test_address_to_u256_conversion() { + // Test that address conversion preserves the address bytes correctly + let addr = Address::repeat_byte(0xAB); + let value = U256::from_be_slice(addr.as_slice()); + + // Address is 20 bytes, should be right-aligned in U256 (32 bytes) + let bytes = value.to_be_bytes::<32>(); + + // First 12 bytes should be zero (padding) + assert_eq!(&bytes[..12], &[0u8; 12]); + + // Last 20 bytes should be the address + assert_eq!(&bytes[12..], addr.as_slice()); + } + + #[test] + fn test_v2_metadata_address_values() { + // Test specific address encoding + let token0 = Address::repeat_byte(0x11); + let token1 = Address::repeat_byte(0x22); + + let metadata = V2PoolMetadata { + token0, + token1, + last_block_timestamp: 0, + }; + + let token0_value = U256::from_be_slice(metadata.token0.as_slice()); + let token1_value = U256::from_be_slice(metadata.token1.as_slice()); + + // Values should be different + assert_ne!(token0_value, token1_value); + + // Each should be non-zero + assert_ne!(token0_value, U256::ZERO); + assert_ne!(token1_value, U256::ZERO); + + // Verify round-trip: extract address bytes back + let token0_bytes = token0_value.to_be_bytes::<32>(); + let token1_bytes = token1_value.to_be_bytes::<32>(); + + assert_eq!(&token0_bytes[12..], token0.as_slice()); + assert_eq!(&token1_bytes[12..], token1.as_slice()); + } // ==================== block context tests ==================== diff --git a/src/cache/overlay.rs b/src/cache/overlay.rs index bedd993..4831517 100644 --- a/src/cache/overlay.rs +++ b/src/cache/overlay.rs @@ -15,8 +15,8 @@ use revm::{ state::{AccountInfo, Bytecode}, }; -use super::CallSimulationResult; use super::snapshot::EvmSnapshot; +use super::{CallSimulationResult, SimStatus, TxConfig}; use crate::access_set::StorageAccessList; use crate::errors::{SimError, SimulationError, SimulationResult}; use crate::inspector::TransferInspector; @@ -61,22 +61,36 @@ impl EvmOverlay { } } - /// Get the chain ID from the underlying snapshot. + /// Chain ID of the block context captured by the underlying snapshot. + /// + /// This is the value installed into `cfg.chain_id` by [`Self::build_evm`]. pub fn chain_id(&self) -> u64 { self.snapshot.chain_id } - /// Get the block number from the underlying snapshot. + /// Block number of the snapshot's block context, or `None` if it was not + /// captured. + /// + /// When present this is the `block.number` simulations run against; when + /// `None`, [`Self::build_evm`] leaves revm's default block number in place. pub fn block_number(&self) -> Option { self.snapshot.block_number } - /// Get the base fee from the underlying snapshot. + /// Base fee of the snapshot's block context, or `None` if it was not + /// captured. + /// + /// Note that base-fee checks are disabled in the simulation EVM, so this is + /// informational rather than enforced against the transaction. pub fn basefee(&self) -> Option { self.snapshot.basefee } - /// Get the timestamp from the underlying snapshot. + /// Timestamp of the snapshot's block context, or `None` if it was not + /// captured. + /// + /// When `None`, [`Self::build_evm`] substitutes the current wall-clock time + /// for `block.timestamp`. pub fn timestamp(&self) -> Option { self.snapshot.timestamp } @@ -141,7 +155,38 @@ impl EvmOverlay { evm } - /// Execute a non-committing call and return the result. + /// Execute a non-committing call and return the raw [`ExecutionResult`]. + /// + /// The EVM state is reverted to a checkpoint after execution on *both* + /// success and failure, so the call never mutates this overlay's dirty + /// layer. Each overlay simulation is therefore isolated: repeated calls all + /// observe the same base state. + /// + /// A revert or halt is *not* an error here — it is reported through the + /// returned [`ExecutionResult`] variant. Only failure to build or transact + /// the call yields `Err`. + /// + /// # Errors + /// + /// Returns an error if the [`TxEnv`] cannot be built from the given inputs, + /// or if revm fails to transact the call (for example a database error + /// while loading state from the RPC fallback). + /// + /// # Examples + /// + /// ```no_run + /// # use std::sync::Arc; + /// # use alloy_primitives::{Address, Bytes}; + /// # use evm_fork_cache::cache::{EvmOverlay, EvmSnapshot}; + /// # fn run(snapshot: Arc) -> anyhow::Result<()> { + /// let mut overlay = EvmOverlay::new(snapshot, None); + /// let result = overlay.call_raw(Address::ZERO, Address::ZERO, Bytes::new())?; + /// // State is reverted; a second call sees the same base state. + /// let _again = overlay.call_raw(Address::ZERO, Address::ZERO, Bytes::new())?; + /// # let _ = result; + /// # Ok(()) + /// # } + /// ``` pub fn call_raw( &mut self, from: Address, @@ -224,9 +269,45 @@ impl EvmOverlay { /// Simulate a call with transfer tracking via the `TransferInspector`. /// - /// This is the overlay-compatible equivalent of `EvmCache::simulate_with_transfer_tracking`. - /// It captures ERC20 Transfer events during execution to compute balance deltas - /// without relying on pre/post balance queries. + /// This is the overlay-compatible equivalent of + /// [`super::EvmCache::simulate_with_transfer_tracking`]. It captures ERC20 + /// Transfer events during execution to compute balance deltas for `owner` + /// (restricted to `tokens` when provided) without relying on pre/post + /// balance queries. + /// + /// On a reverting or halting call the EVM state is reverted to a checkpoint + /// before returning, so a failed simulation never mutates this overlay. On + /// success the call either commits the journaled changes into the overlay's + /// dirty layer (`commit == true`) or reverts them (`commit == false`); a + /// non-committing run leaves each overlay simulation isolated from the next. + /// + /// # Errors + /// + /// Returns an error if the [`TxEnv`] cannot be built, if revm fails to + /// transact the call, if the call reverts (mapped from the revert payload), + /// or if the call halts. In every error case the EVM state is reverted + /// first, regardless of `commit`. + /// + /// # Examples + /// + /// ```no_run + /// # use std::sync::Arc; + /// # use alloy_primitives::{Address, Bytes}; + /// # use evm_fork_cache::cache::{EvmOverlay, EvmSnapshot}; + /// # fn run(snapshot: Arc, token: Address, owner: Address) -> anyhow::Result<()> { + /// let mut overlay = EvmOverlay::new(snapshot, None); + /// let sim = overlay.simulate_with_transfer_tracking( + /// owner, + /// token, + /// Bytes::new(), + /// owner, + /// Some([token]), + /// false, // non-committing: state is reverted afterwards + /// )?; + /// let _delta = sim.token_deltas.get(&token); + /// # Ok(()) + /// # } + /// ``` pub fn simulate_with_transfer_tracking( &mut self, from: Address, @@ -277,6 +358,7 @@ impl EvmOverlay { } Ok(CallSimulationResult { + status: SimStatus::Success, gas_used, token_deltas, logs, @@ -302,18 +384,83 @@ impl EvmOverlay { } } - /// Execute a non-committing call and return the result + access list. + /// Execute a non-committing call and return the result plus the touched + /// [`StorageAccessList`]. + /// + /// The access list is collected from every account marked touched in the + /// journaled state after execution, recording both the touched accounts and + /// the storage slots accessed under each. + /// + /// The EVM state is reverted to a checkpoint after a successful transact on + /// both success and revert/halt outcomes, so the call never mutates this + /// overlay's dirty layer and each overlay simulation stays isolated. As with + /// [`Self::call_raw`], a revert or halt is reported through the returned + /// [`ExecutionResult`] rather than as an error. + /// + /// # Errors + /// + /// Returns an error if the [`TxEnv`] cannot be built, or if revm fails to + /// transact the call (for example a database error while loading state). + /// + /// # Examples + /// + /// ```no_run + /// # use std::sync::Arc; + /// # use alloy_primitives::{Address, Bytes}; + /// # use evm_fork_cache::cache::{EvmOverlay, EvmSnapshot}; + /// # fn run(snapshot: Arc) -> anyhow::Result<()> { + /// let mut overlay = EvmOverlay::new(snapshot, None); + /// let (result, access_list) = + /// overlay.call_raw_with_access_list(Address::ZERO, Address::ZERO, Bytes::new())?; + /// # let _ = (result, access_list); + /// # Ok(()) + /// # } + /// ``` pub fn call_raw_with_access_list( &mut self, from: Address, to: Address, calldata: Bytes, ) -> Result<(ExecutionResult, StorageAccessList)> { - let tx = TxEnv::builder() + self.call_raw_with_access_list_with(from, to, calldata, &TxConfig::default()) + } + + /// Like [`call_raw_with_access_list`](Self::call_raw_with_access_list) but + /// honors a full [`TxConfig`]: native `value`, `gas_limit`, `gas_price`, + /// `nonce`, and a pre-warming EIP-2930 `access_list`. + /// + /// This is what the freshness optimistic loop uses so a [`SimRequest`]'s tx + /// environment — e.g. a payable call carrying `value`, or a gas-bounded call + /// — is reproduced faithfully instead of silently running as a zero-value, + /// default-gas call. Like the shorthand it is non-committing (the checkpoint + /// is reverted) and returns the captured storage access list. + /// + /// [`SimRequest`]: crate::freshness::SimRequest + pub fn call_raw_with_access_list_with( + &mut self, + from: Address, + to: Address, + calldata: Bytes, + tx: &TxConfig, + ) -> Result<(ExecutionResult, StorageAccessList)> { + let mut builder = TxEnv::builder() .caller(from) .kind(TxKind::Call(to)) .data(calldata) - .value(U256::ZERO) + .value(tx.value); + if let Some(gas_limit) = tx.gas_limit { + builder = builder.gas_limit(gas_limit); + } + if let Some(gas_price) = tx.gas_price { + builder = builder.gas_price(gas_price); + } + if let Some(nonce) = tx.nonce { + builder = builder.nonce(nonce); + } + if let Some(access_list) = &tx.access_list { + builder = builder.access_list(access_list.clone()); + } + let tx_env = builder .build() .map_err(|e| anyhow!("Failed to build tx env: {:?}", e))?; @@ -321,7 +468,7 @@ impl EvmOverlay { use revm::context_interface::JournalTr; let checkpoint = evm.journaled_state.checkpoint(); let result = evm - .transact_one(tx) + .transact_one(tx_env) .map_err(|e| anyhow!("Failed to transact: {:?}", e))?; let mut access_list = StorageAccessList::default(); @@ -340,10 +487,35 @@ impl EvmOverlay { /// Write a storage value into this overlay's dirty layer. /// - /// The dirty layer takes precedence over the snapshot on subsequent reads, - /// so this lets a caller (e.g. the freshness validator) inject a fresh slot - /// value into a snapshot-backed overlay before re-running a simulation, - /// without mutating the shared snapshot. + /// The dirty layer takes precedence over the snapshot on subsequent reads + /// (see the lookup order on [`EvmOverlay`]), so this injects a value into a + /// snapshot-backed overlay without mutating the shared snapshot. + /// + /// # Freshness validation + /// + /// This is the freshness validator's correction step. When a slot the + /// snapshot captured is found to be stale, the validator writes the + /// freshly-fetched value here and then re-runs the simulation (e.g. via + /// [`Self::call_raw`]): the re-run reads the corrected slot out of the dirty + /// layer instead of the stale snapshot value, so the corrected result + /// becomes observable. Because the override lives only in this overlay, + /// other overlays sharing the same `Arc` are unaffected. + /// + /// # Examples + /// + /// ```no_run + /// # use std::sync::Arc; + /// # use alloy_primitives::{Address, Bytes, U256}; + /// # use evm_fork_cache::cache::{EvmOverlay, EvmSnapshot}; + /// # fn run(snapshot: Arc, token: Address, slot: U256) -> anyhow::Result<()> { + /// let mut overlay = EvmOverlay::new(snapshot, None); + /// // Inject the fresh value, then re-run to observe the corrected result. + /// overlay.override_slot(token, slot, U256::from(42u64)); + /// let corrected = overlay.call_raw(Address::ZERO, token, Bytes::new())?; + /// # let _ = corrected; + /// # Ok(()) + /// # } + /// ``` pub fn override_slot(&mut self, address: Address, slot: U256, value: U256) { self.dirty_storage .entry(address) diff --git a/src/cache/slot_observations.rs b/src/cache/slot_observations.rs index c88cc63..84c83aa 100644 --- a/src/cache/slot_observations.rs +++ b/src/cache/slot_observations.rs @@ -117,6 +117,44 @@ impl SlotObservationTracker { /// `now` is the current clock value (block number or unix seconds) and /// `params` carries the (clock-unit) thresholds — see /// [`crate::freshness::FreshnessParams`]. + /// + /// The heuristic is fully deterministic (no randomness): a never-observed slot + /// always refetches, as does one with fewer than + /// [`min_observations`](crate::freshness::FreshnessParams::min_observations); + /// once enough observations accrue, a never-changed slot is reused until the + /// [`max_reuse`](crate::freshness::FreshnessParams::max_reuse) window elapses, + /// while changing slots refetch once the probabilistic expected-change estimate + /// crosses [`staleness_threshold`](crate::freshness::FreshnessParams::staleness_threshold). + /// + /// # Examples + /// The deterministic threshold edges around a stable (never-changed) slot: + /// + /// ``` + /// use alloy_primitives::{Address, U256}; + /// use evm_fork_cache::cache::SlotObservationTracker; + /// use evm_fork_cache::freshness::FreshnessParams; + /// + /// let params = FreshnessParams::default(); + /// let mut tracker = SlotObservationTracker::new(); + /// let addr = Address::repeat_byte(0x01); + /// let slot = U256::from(0); + /// + /// // An unobserved slot must always be fetched. + /// assert!(tracker.should_refetch(addr, slot, 0, ¶ms)); + /// + /// // Record fewer than `min_observations` of the same value: still refetches + /// // because there is not enough data to trust the change frequency. + /// for now in 0..(params.min_observations - 1) { + /// tracker.observe(addr, slot, U256::from(42), now as u64); + /// } + /// assert!(tracker.should_refetch(addr, slot, params.min_observations as u64, ¶ms)); + /// + /// // One more identical observation reaches `min_observations`; the slot has + /// // never changed, so within the reuse window it is now reused (no refetch). + /// let last = params.min_observations as u64 - 1; + /// tracker.observe(addr, slot, U256::from(42), last); + /// assert!(!tracker.should_refetch(addr, slot, last, ¶ms)); + /// ``` pub fn should_refetch( &self, addr: Address, diff --git a/src/cache/snapshot.rs b/src/cache/snapshot.rs index 70dee8e..d36364b 100644 --- a/src/cache/snapshot.rs +++ b/src/cache/snapshot.rs @@ -1,10 +1,32 @@ //! Immutable, shareable EVM state snapshots. //! +//! # Flattening model +//! //! A snapshot flattens the live cache (CacheDB overlay plus the BlockchainDb //! backend) into a single immutable, `Send + Sync` view of accounts and -//! storage. Because it is read-only it can be wrapped in an `Arc` and shared +//! storage. The layered lookups of the live cache are collapsed into flat +//! `HashMap`s at creation time, so every read against the snapshot is an O(1) +//! lookup with no locks and no fallback chain. +//! +//! # `Arc` sharing +//! +//! Because the snapshot is read-only it can be wrapped in an `Arc` and shared //! across threads, letting many parallel simulations read from one consistent -//! state while each layers its own writes through a separate overlay. +//! state. Handing a new simulation task its state is a cheap `Arc::clone` +//! rather than a deep copy of the accounts/storage maps. +//! +//! # Per-simulation dirty layer +//! +//! Each simulation does not mutate the shared snapshot. Instead it wraps the +//! `Arc` in an [`EvmOverlay`], which adds a per-simulation +//! *dirty layer* on top: writes (committed account/storage changes, RPC +//! fallbacks, freshness overrides) land in the overlay's own maps and take +//! precedence over the snapshot on subsequent reads. Two overlays built from +//! the same `Arc` are fully isolated from one another, so +//! simulations can run in parallel without contending for or corrupting the +//! shared base state. +//! +//! [`EvmOverlay`]: super::EvmOverlay use std::collections::HashMap; diff --git a/src/cache/storage_keys.rs b/src/cache/storage_keys.rs index 949d482..01117f6 100644 --- a/src/cache/storage_keys.rs +++ b/src/cache/storage_keys.rs @@ -64,6 +64,27 @@ pub const V2_RESERVES_SLOT: U256 = U256::from_limbs([8, 0, 0, 0]); /// /// tickBitmap is a `mapping(int16 => uint256)` at base slot 6. /// For a mapping at slot `p`, the value for key `k` is at `keccak256(abi.encode(k, p))`. +/// +/// This is the convenience wrapper over +/// [`v3_tick_bitmap_storage_key_with_base`] pinned to +/// [`V3_TICK_BITMAP_BASE_SLOT`]. +/// +/// # Examples +/// +/// ``` +/// use evm_fork_cache::cache::{ +/// v3_tick_bitmap_storage_key, v3_tick_bitmap_storage_key_with_base, +/// V3_TICK_BITMAP_BASE_SLOT, +/// }; +/// +/// // Equivalent to calling the `_with_base` form with the default base slot. +/// assert_eq!( +/// v3_tick_bitmap_storage_key(3), +/// v3_tick_bitmap_storage_key_with_base(3, V3_TICK_BITMAP_BASE_SLOT), +/// ); +/// // The key is deterministic and distinct per word position. +/// assert_ne!(v3_tick_bitmap_storage_key(3), v3_tick_bitmap_storage_key(-3)); +/// ``` pub fn v3_tick_bitmap_storage_key(word_position: i16) -> U256 { v3_tick_bitmap_storage_key_with_base(word_position, V3_TICK_BITMAP_BASE_SLOT) } @@ -71,6 +92,22 @@ pub fn v3_tick_bitmap_storage_key(word_position: i16) -> U256 { /// Compute the storage key for a V3-style tickBitmap entry with a custom base slot. /// /// PancakeSwap V3 uses base slot 7 instead of Uniswap V3's slot 6. +/// +/// The key is `keccak256(abi.encode(int256(word_position), base_slot))`, so a +/// different `base_slot` yields a different key for the same word position. +/// +/// # Examples +/// +/// ``` +/// use evm_fork_cache::cache::{ +/// v3_tick_bitmap_storage_key_with_base, V3_TICK_BITMAP_BASE_SLOT, +/// PANCAKE_V3_TICK_BITMAP_BASE_SLOT, +/// }; +/// +/// let uniswap = v3_tick_bitmap_storage_key_with_base(10, V3_TICK_BITMAP_BASE_SLOT); +/// let pancake = v3_tick_bitmap_storage_key_with_base(10, PANCAKE_V3_TICK_BITMAP_BASE_SLOT); +/// assert_ne!(uniswap, pancake); +/// ``` pub fn v3_tick_bitmap_storage_key_with_base(word_position: i16, base_slot: U256) -> U256 { let word_i256 = i256_from_i16(word_position); let mut preimage = [0u8; 64]; @@ -84,13 +121,41 @@ pub fn v3_tick_bitmap_storage_key_with_base(word_position: i16, base_slot: U256) /// The ticks mapping is at slot 5: `mapping(int24 => Tick.Info)` /// Storage key: `keccak256(abi.encode(int256(tick), uint256(5)))` /// The Tick.Info struct occupies 4 consecutive slots starting from the base. +/// +/// This is the convenience wrapper over [`v3_tick_info_storage_keys_with_base`] +/// pinned to [`V3_TICKS_BASE_SLOT`]. +/// +/// # Examples +/// +/// ``` +/// use evm_fork_cache::cache::v3_tick_info_storage_keys; +/// use alloy_primitives::U256; +/// +/// let keys = v3_tick_info_storage_keys(0); +/// // The four slots are consecutive, starting from the hashed base. +/// assert_eq!(keys[1], keys[0] + U256::from(1)); +/// assert_eq!(keys[2], keys[0] + U256::from(2)); +/// assert_eq!(keys[3], keys[0] + U256::from(3)); +/// ``` pub fn v3_tick_info_storage_keys(tick: i32) -> [U256; 4] { v3_tick_info_storage_keys_with_base(tick, V3_TICKS_BASE_SLOT) } /// Compute the storage slot keys for a V3-style tick's Info struct with a custom ticks mapping slot. /// -/// PancakeSwap V3 uses ticks at slot 6 instead of Uniswap V3's slot 5. +/// PancakeSwap V3 uses ticks at slot 6 instead of Uniswap V3's slot 5. The four +/// returned keys are consecutive, starting from +/// `keccak256(abi.encode(int256(tick), ticks_slot))`. +/// +/// # Examples +/// +/// ``` +/// use evm_fork_cache::cache::{v3_tick_info_storage_keys_with_base, V3_TICKS_BASE_SLOT}; +/// use alloy_primitives::U256; +/// +/// let keys = v3_tick_info_storage_keys_with_base(-100, V3_TICKS_BASE_SLOT); +/// assert_eq!(keys[3], keys[0] + U256::from(3)); +/// ``` pub fn v3_tick_info_storage_keys_with_base(tick: i32, ticks_slot: U256) -> [U256; 4] { let tick_i256 = i256_from_i24(tick); let mut preimage = [0u8; 64]; diff --git a/src/cache/tick_snapshot.rs b/src/cache/tick_snapshot.rs index 5823c08..e5f6f5a 100644 --- a/src/cache/tick_snapshot.rs +++ b/src/cache/tick_snapshot.rs @@ -32,6 +32,10 @@ pub struct TickInfo { } /// Serializable tick info for V3 pools. +/// +/// On-disk counterpart of [`TickInfo`] with the same three fields. It exists as +/// a distinct type so the persisted snapshot format can evolve independently of +/// the public [`TickInfo`] used by the simulation API. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SerializableTickInfo { pub liquidity_gross: u128, @@ -60,6 +64,13 @@ pub struct V3PoolTickSnapshot { impl V3PoolTickSnapshot { /// Create a new tick snapshot from pool data. + /// + /// Captures the in-memory `tick_bitmap` and `ticks` maps along with the + /// pool's current `liquidity` and `tick`, converting the integer map keys to + /// their `String` form for serialization. The conversion is total (no entry + /// is dropped); the inverse [`V3PoolTickSnapshot::to_tick_bitmap`] / + /// [`V3PoolTickSnapshot::to_ticks`] may drop entries whose string keys fail + /// to parse. pub fn from_pool_data( tick_bitmap: &std::collections::HashMap, ticks: &std::collections::HashMap, @@ -90,6 +101,11 @@ impl V3PoolTickSnapshot { } /// Convert tick_bitmap back to HashMap. + /// + /// Reverses the `i16 -> String` keying done by + /// [`V3PoolTickSnapshot::from_pool_data`]. Any entry whose string key does + /// not parse back to an `i16` is silently dropped, so a corrupted or + /// out-of-range key produces a smaller map rather than an error. pub fn to_tick_bitmap(&self) -> std::collections::HashMap { self.tick_bitmap .iter() @@ -98,6 +114,11 @@ impl V3PoolTickSnapshot { } /// Convert ticks back to `HashMap`. + /// + /// Reverses the `i32 -> String` keying done by + /// [`V3PoolTickSnapshot::from_pool_data`]. Any entry whose string key does + /// not parse back to an `i32` is silently dropped, so a corrupted or + /// out-of-range key produces a smaller map rather than an error. pub fn to_ticks(&self) -> std::collections::HashMap { self.ticks .iter() @@ -129,6 +150,11 @@ pub struct V3TickSnapshotCache { impl V3TickSnapshotCache { /// Load tick snapshot cache from disk (binary format). + /// + /// Returns `None` if `path` cannot be read or its contents fail to decode as + /// bincode for this type; a decode failure is logged at `warn` level and + /// treated as a cache miss. The format has no version header, so a file from + /// an incompatible build also yields `None`. pub fn load(path: &Path) -> Option { let data = std::fs::read(path).ok()?; bincode::deserialize(&data) @@ -137,6 +163,14 @@ impl V3TickSnapshotCache { } /// Save tick snapshot cache to disk (binary format). + /// + /// Creates the parent directory if needed, then writes the + /// bincode-serialized cache to `path`. + /// + /// # Errors + /// + /// Returns an error if the parent directory cannot be created, if bincode + /// serialization fails, or if writing the file fails. pub fn save(&self, path: &Path) -> Result<()> { if let Some(parent) = path.parent() { std::fs::create_dir_all(parent)?; @@ -152,11 +186,15 @@ impl V3TickSnapshotCache { } /// Store a tick snapshot for a pool. + /// + /// Overwrites any existing snapshot for `address`. pub fn set(&mut self, address: Address, snapshot: V3PoolTickSnapshot) { self.snapshots.insert(address, snapshot); } /// Remove a tick snapshot for a pool. + /// + /// A no-op if no snapshot is stored for `address`. pub fn remove(&mut self, address: Address) { self.snapshots.remove(&address); } diff --git a/src/create3.rs b/src/create3.rs index 40fe7b7..b7d7082 100644 --- a/src/create3.rs +++ b/src/create3.rs @@ -9,7 +9,18 @@ use alloy_primitives::{Address, B256, address, b256, keccak256}; -/// A widely deployed universal CREATE3 factory implementation. +/// Address of the widely deployed universal CREATE3 factory (the CreateX / +/// CREATE3 factory implementation). +/// +/// This is the canonical cross-chain address at which the CreateX-style +/// CREATE3 factory has been deterministically deployed on many EVM networks. +/// The derivation in this module assumes the factory at this address uses the +/// CREATE3 proxy init code whose hash is `CREATE3_PROXY_INITCODE_HASH`. +/// +/// Callers must verify the factory is actually deployed at this address on +/// their target chain before relying on a derived address: if the factory is +/// absent (or a chain hosts a different factory implementation), the derived +/// address will not correspond to any real deployment. pub const UNIVERSAL_CREATE3_FACTORY: Address = address!("93FEC2C00BfE902F733B57c5a6CeeD7CD1384AE1"); // CREATE3 proxy initcode used by the universal factory implementation. @@ -19,10 +30,54 @@ const CREATE3_PROXY_INITCODE_HASH: B256 = /// Derive CREATE3 deployment address for the universal factory implementation. /// +/// CREATE3 deploys in two hops: the factory first `CREATE2`-deploys a tiny +/// fixed proxy, then that proxy `CREATE`s the actual contract as its first +/// (nonce-1) deployment. Because both hops use only the factory, the salt, and +/// a fixed proxy init code, the final address depends solely on `factory`, +/// `deployer`, and `salt` — it is **independent of the deployed contract's +/// bytecode**. Two different contracts deployed with the same inputs land at +/// the same address. +/// /// Formula: -/// 1) mixedSalt = keccak256(abi.encodePacked(deployer, salt)) -/// 2) proxy = create2(factory, mixedSalt, CREATE3_PROXY_INITCODE_HASH) -/// 3) deployed = address(keccak256(rlp([proxy, 1]))) +/// 1. `mixedSalt = keccak256(abi.encodePacked(deployer, salt))` — binds the +/// salt to the logical deployer. +/// 2. `proxy = create2(factory, mixedSalt, CREATE3_PROXY_INITCODE_HASH)` — +/// the CREATE2 address of the proxy. `CREATE3_PROXY_INITCODE_HASH` is the +/// keccak256 of the fixed proxy init code, so the proxy address is fully +/// determined by the factory and mixed salt. +/// 3. `deployed = address(keccak256(rlp([proxy, 1])))` — the CREATE address of +/// the proxy's first deployment (nonce 1). The RLP framing bytes encode the +/// short list `[proxy, 1]`: `0xd6` is the RLP list header for the 22-byte +/// payload that follows, `0x94` introduces the 20-byte `proxy` address, and +/// `0x01` is the RLP encoding of the proxy's nonce (1), since a fresh +/// contract account's first `CREATE` uses nonce 1. +/// +/// The address is returned as the low 20 bytes of each keccak256 hash, matching +/// the EVM's address-from-hash convention. +/// +/// `factory` lets you derive against a non-canonical factory deployment; for +/// the canonical address use [`derive_universal_create3_address`]. +/// +/// ``` +/// use evm_fork_cache::create3::derive_create3_address; +/// use alloy_primitives::{Address, B256, address, b256}; +/// +/// let factory: Address = address!("93FEC2C00BfE902F733B57c5a6CeeD7CD1384AE1"); +/// let deployer: Address = address!("00000000000000000000000000000000000000aa"); +/// let salt: B256 = +/// b256!("1111111111111111111111111111111111111111111111111111111111111111"); +/// +/// // The derivation is a pure function of (factory, deployer, salt): identical +/// // inputs always yield the same address. +/// let a = derive_create3_address(factory, deployer, salt); +/// let b = derive_create3_address(factory, deployer, salt); +/// assert_eq!(a, b); +/// +/// // Changing the salt changes the derived address. +/// let other_salt: B256 = +/// b256!("2222222222222222222222222222222222222222222222222222222222222222"); +/// assert_ne!(a, derive_create3_address(factory, deployer, other_salt)); +/// ``` pub fn derive_create3_address(factory: Address, deployer: Address, salt: B256) -> Address { let mut mixed_salt_input = [0u8; 52]; mixed_salt_input[..20].copy_from_slice(deployer.as_slice()); @@ -48,6 +103,26 @@ pub fn derive_create3_address(factory: Address, deployer: Address, salt: B256) - } /// Derive CREATE3 deployment address via the universal factory. +/// +/// Convenience wrapper around [`derive_create3_address`] that uses +/// [`UNIVERSAL_CREATE3_FACTORY`] as the factory. As with the general form, the +/// result depends only on `deployer` and `salt`, not on the deployed bytecode, +/// and is only meaningful on chains where that factory is actually deployed. +/// +/// ``` +/// use evm_fork_cache::create3::derive_universal_create3_address; +/// use alloy_primitives::{Address, B256, address, b256}; +/// +/// let deployer: Address = address!("00000000000000000000000000000000000000aa"); +/// let salt: B256 = +/// b256!("1111111111111111111111111111111111111111111111111111111111111111"); +/// +/// // Deterministic: the same (deployer, salt) always derive the same address. +/// assert_eq!( +/// derive_universal_create3_address(deployer, salt), +/// derive_universal_create3_address(deployer, salt), +/// ); +/// ``` pub fn derive_universal_create3_address(deployer: Address, salt: B256) -> Address { derive_create3_address(UNIVERSAL_CREATE3_FACTORY, deployer, salt) } diff --git a/src/deploy.rs b/src/deploy.rs index dcc43c7..e7c1d5c 100644 --- a/src/deploy.rs +++ b/src/deploy.rs @@ -32,6 +32,13 @@ impl FoundryArtifact { /// /// The legacy direct string shape `{ "bytecode": "0x..." }` is also /// accepted to make tests and generated artifacts easier to reuse. + /// + /// # Errors + /// + /// Returns an error if the file cannot be read, is not valid JSON, lacks a + /// usable `bytecode`/`bytecode.object` field, or contains bytecode that is + /// empty, not valid hex, or still has unresolved library placeholders (see + /// [`load_foundry_creation_code`]). pub fn load(path: impl AsRef) -> Result { let path = path.as_ref(); let creation_code = load_foundry_creation_code(path)?; @@ -52,6 +59,18 @@ impl FoundryArtifact { /// /// `constructor_args` must already be ABI encoded. Use /// [`encode_constructor_args`] for ordinary Solidity constructor tuples. + /// + /// # Errors + /// + /// Returns an error if the `CREATE` transaction reverts or halts, or if the + /// deployment otherwise fails to produce a deployed address (see + /// [`EvmCache::deploy_contract`]). + /// + /// # Panics + /// + /// Like any method that may fetch missing state, this must run on a + /// multi-thread tokio runtime; deploying on a current-thread runtime panics + /// when the fork DB attempts a synchronous RPC fetch. pub fn deploy( &self, cache: &mut EvmCache, @@ -77,6 +96,21 @@ impl FoundryArtifact { /// constructor-initialized immutables: the temporary deployment computes the /// final runtime bytecode, and `target` keeps its existing storage, balance, /// and nonce. `target` must already have non-empty runtime bytecode. + /// + /// On any error the cache is restored to its pre-deploy snapshot, so a + /// failed etch leaves no partial deployment behind. + /// + /// # Errors + /// + /// Returns an error if `target` is missing or has no runtime bytecode, if + /// the deployment reverts or halts (see [`Self::deploy`]), or if copying the + /// runtime bytecode to `target` fails. + /// + /// # Panics + /// + /// Must run on a multi-thread tokio runtime; the underlying deployment + /// panics on a current-thread runtime when the fork DB attempts a + /// synchronous RPC fetch. pub fn etch( &self, cache: &mut EvmCache, @@ -98,6 +132,21 @@ impl FoundryArtifact { /// /// Use this only for synthetic simulation addresses where there is no /// storage, balance, or nonce to preserve. + /// + /// On any error the cache is restored to its pre-deploy snapshot, so a + /// failed etch leaves no synthetic target account behind. + /// + /// # Errors + /// + /// Returns an error if the deployment reverts or halts (see + /// [`Self::deploy`]), or if copying the runtime bytecode to `target` fails + /// (for example when the deployed contract has empty runtime bytecode). + /// + /// # Panics + /// + /// Must run on a multi-thread tokio runtime; the underlying deployment + /// panics on a current-thread runtime when the fork DB attempts a + /// synchronous RPC fetch. pub fn etch_or_create( &self, cache: &mut EvmCache, @@ -197,9 +246,26 @@ pub struct EtchedContract { /// ABI-encode constructor arguments. /// -/// Pass a Rust tuple matching the constructor parameter list. This mirrors -/// Solidity constructor parameter encoding (`abi.encode(arg0, arg1, ...)`) and -/// avoids the nested tuple encoding produced by `abi.encode((...))`. +/// Pass a tuple of alloy Solidity values matching the constructor parameter +/// list, e.g. `(owner, weth, vault)`. Single-argument constructors need a +/// trailing comma so the value is still a tuple: `(owner,)`. An empty tuple +/// `()` encodes to empty bytes, which is correct for argument-less +/// constructors. +/// +/// The encoding mirrors Solidity constructor parameter encoding +/// (`abi.encode(arg0, arg1, ...)`): it uses [`SolValue::abi_encode_params`], +/// which lays the arguments out as a flat parameter list. This differs from +/// [`SolValue::abi_encode`], which would wrap a tuple in an extra layer +/// (matching `abi.encode((...))`) and produce the wrong bytes for a +/// constructor. +/// +/// The trait bounds spell out "any alloy Solidity value tuple": `T: SolValue` +/// means each element implements the alloy Solidity-value trait, and the +/// `TokenSeq` bound on `T::SolType` requires the tuple's token to be a +/// sequence so it can be encoded as a parameter list. In practice you do not +/// construct these bounds yourself — they are satisfied automatically by +/// tuples of alloy primitives such as [`Address`], [`U256`](alloy_primitives::U256), +/// and `String`. /// /// ```ignore /// let args = evm_fork_cache::deploy::encode_constructor_args((owner, weth, vault)); @@ -213,6 +279,19 @@ where } /// Load creation bytecode from a Foundry artifact. +/// +/// Reads the JSON at `path` and decodes the creation bytecode from +/// `bytecode.object` (or the legacy direct-string `bytecode` field). +/// +/// # Errors +/// +/// Returns an error when: +/// - the file cannot be read, +/// - the contents are not valid JSON, +/// - the JSON has no `bytecode` field, or `bytecode` has neither an `object` +/// string nor a direct string value, +/// - the bytecode hex is empty, still contains unresolved library +/// placeholders (`__$...$__`), or is otherwise not valid hex. pub fn load_foundry_creation_code(path: impl AsRef) -> Result { let path = path.as_ref(); let content = std::fs::read_to_string(path) @@ -244,6 +323,19 @@ pub fn load_foundry_creation_code(path: impl AsRef) -> Result { } /// Build init code from creation bytecode and ABI-encoded constructor args. +/// +/// Init code is simply the contract's creation bytecode with the ABI-encoded +/// constructor arguments appended, matching how the EVM expects a `CREATE` +/// payload to be laid out. The `constructor_args` must already be ABI encoded; +/// use [`encode_constructor_args`] to produce them from an alloy Solidity +/// value tuple. +/// +/// ``` +/// use evm_fork_cache::deploy::build_init_code; +/// +/// let init = build_init_code([0x60, 0x80], [0x01, 0x02, 0x03]); +/// assert_eq!(init.as_ref(), &[0x60, 0x80, 0x01, 0x02, 0x03]); +/// ``` pub fn build_init_code( creation_code: impl AsRef<[u8]>, constructor_args: impl AsRef<[u8]>, @@ -258,6 +350,17 @@ pub fn build_init_code( /// Deploy a Foundry artifact into the forked EVM and return its temporary /// deployed address. +/// +/// # Errors +/// +/// Returns an error if the artifact cannot be loaded (see +/// [`load_foundry_creation_code`]) or if the deployment reverts or halts (see +/// [`FoundryArtifact::deploy`]). +/// +/// # Panics +/// +/// Must run on a multi-thread tokio runtime; the deployment panics on a +/// current-thread runtime when the fork DB attempts a synchronous RPC fetch. pub fn deploy_foundry_artifact( cache: &mut EvmCache, artifact_path: impl AsRef, @@ -273,6 +376,18 @@ pub fn deploy_foundry_artifact( /// and nonce are preserved. If `target` is missing or has no runtime bytecode, /// this returns an error. Use [`etch_foundry_artifact_or_create`] for synthetic /// simulation addresses. +/// +/// # Errors +/// +/// Returns an error if the artifact cannot be loaded (see +/// [`load_foundry_creation_code`]), if `target` is missing or has no runtime +/// bytecode, if the deployment reverts or halts, or if copying the runtime +/// bytecode to `target` fails (see [`FoundryArtifact::etch`]). +/// +/// # Panics +/// +/// Must run on a multi-thread tokio runtime; the deployment panics on a +/// current-thread runtime when the fork DB attempts a synchronous RPC fetch. pub fn etch_foundry_artifact( cache: &mut EvmCache, target: Address, @@ -288,6 +403,19 @@ pub fn etch_foundry_artifact( /// /// Prefer [`etch_foundry_artifact`] for forked/live contract addresses whose /// storage, balance, or nonce should be preserved. +/// +/// # Errors +/// +/// Returns an error if the artifact cannot be loaded (see +/// [`load_foundry_creation_code`]), if the deployment reverts or halts, or if +/// copying the runtime bytecode to `target` fails, for example when the +/// deployed contract has empty runtime bytecode (see +/// [`FoundryArtifact::etch_or_create`]). +/// +/// # Panics +/// +/// Must run on a multi-thread tokio runtime; the deployment panics on a +/// current-thread runtime when the fork DB attempts a synchronous RPC fetch. pub fn etch_foundry_artifact_or_create( cache: &mut EvmCache, target: Address, diff --git a/src/errors.rs b/src/errors.rs index f93eeff..8a0c229 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -10,6 +10,11 @@ //! Application-specific selectors therefore live in the application, not in this //! generic layer: define them with `sol!` and register them once. //! +//! Note that [`Panic(uint256)`](RevertReason::Panic) codes that exceed +//! `u64::MAX` are dropped to `None` during decoding (and so surface as +//! [`RevertReason::Unknown`]). This is benign: real compiler-emitted panic +//! codes are single-byte constants (e.g. `0x11`, `0x32`). +//! //! ``` //! use alloy_sol_types::{SolError, sol}; //! use evm_fork_cache::errors::{RevertDecoder, RevertReason}; @@ -37,10 +42,12 @@ use std::sync::{Arc, OnceLock}; use alloy_primitives::{Bytes, FixedBytes}; use alloy_sol_types::SolError; -/// 4-byte selector of the standard Solidity `Error(string)` revert. +/// 4-byte selector of the standard Solidity `Error(string)` revert +/// (`0x08c379a0`), emitted by `require`/`revert("msg")`. pub const ERROR_SELECTOR: [u8; 4] = [0x08, 0xc3, 0x79, 0xa0]; -/// 4-byte selector of the standard Solidity `Panic(uint256)` revert. +/// 4-byte selector of the standard Solidity `Panic(uint256)` revert +/// (`0x4e487b71`), emitted on overflow, division-by-zero, etc. pub const PANIC_SELECTOR: [u8; 4] = [0x4e, 0x48, 0x7b, 0x71]; /// A decoded contract-defined custom error. @@ -48,7 +55,8 @@ pub const PANIC_SELECTOR: [u8; 4] = [0x4e, 0x48, 0x7b, 0x71]; pub struct CustomRevert { /// Human-readable signature, e.g. `"Unauthorized(address)"`. pub name: Cow<'static, str>, - /// The error's 4-byte selector. + /// The error's 4-byte selector (the first 4 bytes of [`data`](Self::data)), + /// the `keccak256` prefix of [`name`](Self::name). pub selector: FixedBytes<4>, /// Debug-formatted decoded parameters, when the body decoded successfully. /// @@ -71,13 +79,16 @@ impl fmt::Display for CustomRevert { /// A decoded EVM revert reason. #[derive(Debug, Clone, PartialEq, Eq)] pub enum RevertReason { - /// The call reverted with no return data. + /// The call reverted with no return data (e.g. a bare `revert()` or `assert` + /// in older Solidity, or an empty `require`). Empty, /// Standard Solidity `Error(string)` revert (e.g. `require(cond, "msg")`). Error(String), /// Standard Solidity `Panic(uint256)` revert (e.g. arithmetic overflow). Panic(u64), - /// A registered contract-defined custom error. + /// A contract-defined custom error whose selector was registered on the + /// decoder via [`RevertDecoder::with_error`], [`RevertDecoder::register`], + /// or [`RevertDecoder::register_raw`]. Custom(CustomRevert), /// A selector that matched no built-in or registered custom error. Unknown { @@ -137,7 +148,15 @@ impl fmt::Debug for RevertDecoder { } impl RevertDecoder { - /// Create a decoder that recognizes only the standard Solidity built-ins. + /// Create a decoder that recognizes only the standard Solidity built-ins + /// (`Error(string)` and `Panic(uint256)`) and no custom errors. + /// + /// ``` + /// use evm_fork_cache::errors::RevertDecoder; + /// + /// let decoder = RevertDecoder::new(); + /// assert!(decoder.is_empty()); + /// ``` pub fn new() -> Self { Self::default() } @@ -194,6 +213,41 @@ impl RevertDecoder { /// when the selector and signature come from an ABI loaded at runtime. The /// `decode` closure receives the full revert bytes (selector included) and /// returns the formatted parameters, or `None` if it cannot decode them. + /// + /// If the closure returns `None`, the selector still matches: the decode + /// yields a [`RevertReason::Custom`] whose + /// [`params`](CustomRevert::params) is `None`. If an error with the same + /// selector is already registered it is replaced. + /// + /// ``` + /// use alloy_primitives::Bytes; + /// use evm_fork_cache::errors::{RevertDecoder, RevertReason}; + /// + /// let mut decoder = RevertDecoder::new(); + /// // A closure that decodes the parameters when there is a payload byte, + /// // and otherwise reports a decode failure by returning `None`. + /// decoder.register_raw([0xde, 0xad, 0xbe, 0xef], "MyError(uint256)", |data| { + /// (data.len() > 4).then(|| format!("payload {} bytes", data.len() - 4)) + /// }); + /// + /// // Selector plus a payload byte: the closure decodes the parameters. + /// let with_params = Bytes::from(vec![0xde, 0xad, 0xbe, 0xef, 0x00]); + /// match decoder.decode(&with_params) { + /// RevertReason::Custom(custom) => { + /// assert_eq!(custom.name, "MyError(uint256)"); + /// assert_eq!(custom.params.as_deref(), Some("payload 1 bytes")); + /// } + /// other => panic!("expected Custom, got {other}"), + /// } + /// + /// // Bare selector: the closure returns `None`, but the selector still + /// // matches, so the result is a `Custom` with `params == None`. + /// let bare = Bytes::from(vec![0xde, 0xad, 0xbe, 0xef]); + /// match decoder.decode(&bare) { + /// RevertReason::Custom(custom) => assert!(custom.params.is_none()), + /// other => panic!("expected Custom, got {other}"), + /// } + /// ``` pub fn register_raw( &mut self, selector: [u8; 4], @@ -210,20 +264,64 @@ impl RevertDecoder { self } - /// Number of registered custom errors (built-ins are not counted). + /// Number of registered custom errors. The two Solidity built-ins are + /// always recognized and are not counted, so a freshly + /// [`new`](RevertDecoder::new) decoder reports `0`. pub fn len(&self) -> usize { self.custom.len() } - /// Returns `true` if no custom errors are registered. + /// Returns `true` if no custom errors are registered. The built-ins are + /// always recognized regardless, so this is `true` for a freshly + /// [`new`](RevertDecoder::new) decoder. pub fn is_empty(&self) -> bool { self.custom.is_empty() } /// Decode raw EVM revert data into a [`RevertReason`]. /// - /// Resolution order: the two Solidity built-ins, then registered custom - /// errors by selector, then [`RevertReason::Unknown`] for anything else. + /// Resolution order: the two Solidity built-ins (`Error(string)` and + /// `Panic(uint256)`), then registered custom errors by selector, then + /// [`RevertReason::Unknown`] for anything else. Empty input decodes to + /// [`RevertReason::Empty`], and data shorter than 4 bytes decodes to + /// [`RevertReason::Unknown`] with the selector right-padded with zeros. + /// + /// ``` + /// use alloy_primitives::{Bytes, U256}; + /// use alloy_sol_types::{Panic, SolError, sol}; + /// use evm_fork_cache::errors::{RevertDecoder, RevertReason, ERROR_SELECTOR}; + /// + /// sol! { + /// #[derive(Debug)] + /// error Custom(); + /// } + /// + /// let decoder = RevertDecoder::new().with_error::(); + /// + /// // Built-in `Error(string)` decodes natively, without registration. + /// // Layout: selector | offset(0x20) | length | utf8 bytes (padded). + /// let mut bytes = ERROR_SELECTOR.to_vec(); + /// bytes.extend_from_slice(&{ let mut o = [0u8; 32]; o[31] = 0x20; o }); // offset + /// bytes.extend_from_slice(&{ let mut l = [0u8; 32]; l[31] = 2; l }); // length 2 + /// bytes.extend_from_slice(b"hi"); + /// bytes.extend_from_slice(&[0u8; 30]); // pad to 32 + /// assert_eq!(decoder.decode(&Bytes::from(bytes)), RevertReason::Error("hi".into())); + /// + /// // Built-in `Panic(uint256)` decodes natively too. + /// let panic = Bytes::from(Panic { code: U256::from(0x11) }.abi_encode()); + /// assert_eq!(decoder.decode(&panic), RevertReason::Panic(0x11)); + /// + /// // A registered selector resolves to `Custom`. + /// let raw = Bytes::from(Custom::SELECTOR.to_vec()); + /// match decoder.decode(&raw) { + /// RevertReason::Custom(err) => assert_eq!(err.name, "Custom()"), + /// other => panic!("expected Custom, got {other}"), + /// } + /// + /// // An unregistered selector falls through to `Unknown`. + /// let unknown = Bytes::from(vec![0xde, 0xad, 0xbe, 0xef]); + /// assert!(matches!(decoder.decode(&unknown), RevertReason::Unknown { .. })); + /// ``` pub fn decode(&self, data: &Bytes) -> RevertReason { if data.is_empty() { return RevertReason::Empty; @@ -302,11 +400,12 @@ fn decode_solidity_error_string(data: &Bytes) -> Option { /// A structured simulation revert with its decoded reason. #[derive(Debug, Clone)] pub struct SimulationError { - /// Gas used before the revert. + /// Gas consumed before the revert. pub gas_used: u64, - /// Raw revert data returned by the EVM. + /// Raw revert data returned by the EVM (the bytes that were decoded into + /// [`reason`](Self::reason)). pub revert_data: Bytes, - /// The decoded revert reason. + /// The revert reason decoded from [`revert_data`](Self::revert_data). pub reason: RevertReason, } @@ -333,7 +432,8 @@ impl SimulationError { } } - /// The decoded revert reason. + /// The decoded revert reason. Equivalent to borrowing the public + /// [`reason`](Self::reason) field. pub fn reason(&self) -> &RevertReason { &self.reason } @@ -371,7 +471,8 @@ impl SimulationError { } } - /// `true` if the call reverted with no return data. + /// `true` if the call reverted with no return data, i.e. the reason is + /// [`RevertReason::Empty`]. pub fn is_empty_revert(&self) -> bool { matches!(self.reason, RevertReason::Empty) } @@ -389,7 +490,9 @@ impl fmt::Display for SimulationError { impl std::error::Error for SimulationError {} -/// Result type for simulations that distinguish EVM reverts from host errors. +/// Result type returned by simulation entry points: `Ok(T)` on success, or a +/// [`SimError`] distinguishing a transaction-level revert, an EVM halt, and a +/// host-side failure. pub type SimulationResult = Result; /// Error returned by simulation entry points. @@ -398,6 +501,11 @@ pub type SimulationResult = Result; /// [`Revert`](SimError::Revert) (with a decoded reason), an EVM /// [`Halt`](SimError::Halt) (e.g. out of gas), and a host-side /// [`Other`](SimError::Other) failure (RPC, database, ABI encoding). +/// +/// Note that when a revert decodes to [`RevertReason::Panic`], panic codes +/// exceeding `u64::MAX` are dropped to `None` and so surface as +/// [`RevertReason::Unknown`] rather than `Panic`. This is benign: real +/// compiler-emitted panic codes are single-byte constants. #[derive(Debug, thiserror::Error)] pub enum SimError { /// The transaction reverted; carries the decoded revert. @@ -418,17 +526,21 @@ pub enum SimError { } impl SimError { - /// `true` if this is a transaction-level revert. + /// `true` if this is a transaction-level revert, i.e. the + /// [`Revert`](SimError::Revert) variant. pub fn is_revert(&self) -> bool { matches!(self, SimError::Revert(_)) } - /// `true` if the EVM halted (e.g. out of gas). + /// `true` if the EVM halted without returning revert data (e.g. out of + /// gas), i.e. the [`Halt`](SimError::Halt) variant. pub fn is_halt(&self) -> bool { matches!(self, SimError::Halt { .. }) } - /// The decoded revert, if this error is a revert. + /// The decoded [`SimulationError`] if this is a + /// [`Revert`](SimError::Revert), or `None` for a + /// [`Halt`](SimError::Halt) or [`Other`](SimError::Other) error. pub fn as_revert(&self) -> Option<&SimulationError> { match self { SimError::Revert(e) => Some(e), diff --git a/src/freshness.rs b/src/freshness.rs index be98df4..302c301 100644 --- a/src/freshness.rs +++ b/src/freshness.rs @@ -52,17 +52,18 @@ //! ``` use std::collections::{HashMap, HashSet}; -use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::time::{SystemTime, UNIX_EPOCH}; +use alloy_eips::BlockId; use alloy_eips::eip2930::AccessList; use alloy_primitives::{Address, Bytes, U256}; use revm::context::result::ExecutionResult; use tokio::task::JoinHandle; use crate::cache::{ - CallSimulationResult, EvmCache, EvmOverlay, EvmSnapshot, SlotObservationTracker, + CallSimulationResult, EvmCache, EvmOverlay, EvmSnapshot, SimStatus, SlotObservationTracker, StorageBatchFetchFn, TxConfig, }; @@ -323,6 +324,11 @@ impl FreshnessClock for BlockClock { } /// Wall-clock clock: [`now`](FreshnessClock::now) returns unix seconds. +/// +/// A zero-sized unit struct: unlike [`BlockClock`] it holds no `Arc`/`AtomicU64`, +/// since the value is read straight from the system clock on each call. It +/// advances on its own, so [`advance`](FreshnessClock::advance) is the trait +/// default no-op and has no effect. #[derive(Clone, Copy, Debug, Default)] pub struct WallClock; @@ -393,10 +399,12 @@ impl FreshnessPolicy for NeverVerify { } /// Adaptive policy: verifies candidates the observation tracker flags via -/// [`should_refetch`](crate::cache::SlotObservationTracker::should_refetch). +/// [`SlotObservationTracker::should_refetch`](crate::cache::SlotObservationTracker::should_refetch), +/// driven by the thresholds in [`FreshnessParams`]. #[derive(Clone, Debug, Default)] pub struct ObservationDriven { - /// Thresholds for the underlying `should_refetch` heuristic. + /// Thresholds for the underlying [`SlotObservationTracker::should_refetch`](crate::cache::SlotObservationTracker::should_refetch) + /// heuristic. pub params: FreshnessParams, } @@ -513,6 +521,24 @@ impl SimRequest { self.tx.access_list = Some(access_list); self } + + /// Set the native value (wei) sent with the call (e.g. for a payable call). + pub fn with_value(mut self, value: U256) -> Self { + self.tx.value = value; + self + } + + /// Set the gas limit for the call (e.g. to model out-of-gas behavior). + pub fn with_gas_limit(mut self, gas_limit: u64) -> Self { + self.tx.gas_limit = Some(gas_limit); + self + } + + /// Set the gas price (wei) for the call. + pub fn with_gas_price(mut self, gas_price: u128) -> Self { + self.tx.gas_price = Some(gas_price); + self + } } /// Optimistic simulation results plus a handle to their deferred validation. @@ -520,12 +546,27 @@ impl SimRequest { /// Returned by [`FreshnessController::run`] as soon as the optimistic sims /// finish (without awaiting RPC). Read [`optimistic`](Self::optimistic) /// immediately, then `await` [`validate`](Self::validate) for the verdict. -/// Dropping the handle aborts the background validation task. +/// +/// # Cancellation (best-effort) +/// Dropping this — or calling [`into_optimistic`](Self::into_optimistic) — sets a +/// cancel flag and aborts the background task. Cancellation is **cooperative and +/// best-effort, not instantaneous**: `run_validator` is synchronous, so an abort +/// cannot preempt it once it is running. Instead the validator checks the flag at +/// a few checkpoints — before fetching, and before recording observations or +/// queuing a correction — so a cancel observed at a checkpoint prevents the +/// remaining side effects. A validator already executing a synchronous step (e.g. +/// mid-fetch) completes that step before reaching the next checkpoint. The intent +/// is that a dropped speculation does not flow its corrections back into the +/// cache; it does not guarantee that an in-flight fetch is interrupted. pub struct SpeculativeSim { optimistic: Vec, /// `Option` so `validate`/`into_optimistic` can take the handle and skip the /// abort-on-drop; `Drop` only aborts a handle still left in place. validation: Option>, + /// Set when the caller drops or [`into_optimistic`](Self::into_optimistic)s + /// this handle; the validator polls it at its checkpoints to bail out before + /// causing side effects (fetching, observing, queuing corrections). + cancelled: Arc, } impl SpeculativeSim { @@ -536,7 +577,17 @@ impl SpeculativeSim { /// Consume the handle and return the optimistic results, aborting the /// background validation task. + /// + /// # Panics + /// The validation [`JoinHandle`] is single-consumption. Because this takes + /// `self` by value, it and [`validate`](Self::validate) are mutually + /// exclusive: only one of them can ever run for a given `SpeculativeSim`, and + /// each takes the handle. `into_optimistic` takes the handle defensively (it + /// does not panic if the handle is already gone), whereas `validate` panics + /// with `"validation handle taken twice"` if it is invoked once the handle has + /// been consumed. pub fn into_optimistic(mut self) -> Vec { + self.cancelled.store(true, Ordering::Relaxed); if let Some(handle) = self.validation.take() { handle.abort(); } @@ -545,8 +596,17 @@ impl SpeculativeSim { /// Await the deferred validation verdict. /// - /// If the background task panicked or was cancelled, returns - /// [`Validation::Unverified`]. + /// If the background task failed to complete (e.g. it panicked), returns + /// [`Validation::Unverified`]. This consumes `self`, so it is mutually + /// exclusive with the cancel paths ([`into_optimistic`](Self::into_optimistic) + /// / drop) — a handle that is awaited here is never cancelled. + /// + /// # Panics + /// The validation [`JoinHandle`] is single-consumption: it is taken by the + /// first of `validate` or [`into_optimistic`](Self::into_optimistic) to run. + /// `validate` panics with `"validation handle taken twice"` if the handle has + /// already been consumed. Both take `self` by value, so under normal ownership + /// this is unreachable. pub async fn validate(mut self) -> Validation { let handle = self .validation @@ -563,6 +623,7 @@ impl SpeculativeSim { impl Drop for SpeculativeSim { fn drop(&mut self) { + self.cancelled.store(true, Ordering::Relaxed); if let Some(handle) = self.validation.take() { handle.abort(); } @@ -601,7 +662,12 @@ pub struct FreshnessController { } impl FreshnessController { - /// Build a controller with the default [`BlockClock`]. + /// Build a controller with the default [`BlockClock`] (starting at block 0). + /// + /// Starts with a fresh, empty [`SlotObservationTracker`] and an empty + /// pending-corrections queue. Use [`with_tracker`](Self::with_tracker) to share + /// a persisted tracker, or [`with_clock`](Self::with_clock) for a non-default + /// clock such as [`WallClock`]. pub fn new(registry: FreshnessRegistry, policy: P) -> Self { Self::with_clock(registry, policy, BlockClock::new()) } @@ -609,6 +675,11 @@ impl FreshnessController { impl FreshnessController { /// Build a controller with an explicit clock. + /// + /// Starts with a fresh, empty [`SlotObservationTracker`] and an empty + /// pending-corrections queue. The clock's units must match those the + /// `policy`'s [`FreshnessParams`] were tuned for (block numbers for + /// [`BlockClock`], unix seconds for [`WallClock`]). pub fn with_clock(registry: FreshnessRegistry, policy: P, clock: C) -> Self { Self { registry, @@ -621,6 +692,12 @@ impl FreshnessController { } /// Use an existing shared observation tracker (e.g. a persisted one). + /// + /// Builder-style override that replaces the fresh tracker installed by + /// [`new`](Self::new) / [`with_clock`](Self::with_clock) with the given shared + /// handle, so change-frequency history survives across runs or is shared with + /// other components. The background validator updates this same tracker under + /// its `Mutex`. pub fn with_tracker(mut self, tracker: Arc>) -> Self { self.tracker = tracker; self @@ -680,6 +757,18 @@ impl FreshnessController { /// verify. /// 5. Spawn the background validator (Send data only) and return a /// [`SpeculativeSim`] immediately. + /// + /// # Panics + /// Spawns a background task whose (synchronous) fetcher uses + /// `tokio::task::block_in_place` internally, so it must run on a + /// **multi-thread** tokio runtime (`#[tokio::main(flavor = "multi_thread")]` + /// or `Builder::new_multi_thread()`). On a current-thread runtime + /// `block_in_place` panics, mirroring the [`EvmCache`] constructor note. + /// + /// # Errors + /// Returns an error if any optimistic simulation fails to execute against the + /// freshly-created snapshot (propagated from + /// `EvmOverlay::call_raw_with_access_list`). pub fn run( &mut self, cache: &mut EvmCache, @@ -693,22 +782,29 @@ impl FreshnessController { if !pending.is_empty() { let injects: Vec<(Address, U256, U256)> = pending.iter().map(|c| (c.address, c.slot, c.new)).collect(); - cache.inject_storage_batch(&injects); + cache.inject_storage_batch_fresh(&injects); pending.clear(); } } - // 2. Snapshot + fetcher (Arc clones, both Send). + // 2. Snapshot + fetcher (Arc clones, both Send). Capture the cache's + // pinned block now, so the deferred validator fetches at the block the + // snapshot was built from even if the cache is re-pinned meanwhile. let snapshot = cache.create_snapshot(); let fetcher = cache.storage_batch_fetcher().cloned(); + let validation_block = cache.block(); // 3. Optimistic sims + per-sim actual volatile read sets. let mut optimistic = Vec::with_capacity(requests.len()); let mut read_sets: Vec> = Vec::with_capacity(requests.len()); for req in &requests { let mut overlay = EvmOverlay::new(Arc::clone(&snapshot), None); - let (result, access) = - overlay.call_raw_with_access_list(req.from, req.to, req.calldata.clone())?; + let (result, access) = overlay.call_raw_with_access_list_with( + req.from, + req.to, + req.calldata.clone(), + &req.tx, + )?; optimistic.push(result_to_sim(result, &access.to_eip2930())); let volatile: Vec<(Address, U256)> = access @@ -746,12 +842,13 @@ impl FreshnessController { let pending = Arc::clone(&self.pending); let rerun_count = Arc::clone(&self.rerun_count); let optimistic_for_task = optimistic.clone(); + let cancelled = Arc::new(AtomicBool::new(false)); + let cancelled_for_task = Arc::clone(&cancelled); let validation = tokio::spawn(async move { - // Yield once before doing any work. `run_validator` is fully - // synchronous (no `.await` inside), so without an early await point - // an abort-on-drop could not preempt it once polled. Yielding gives - // `into_optimistic`/`Drop` a deterministic chance to cancel the task - // before it touches the tracker or queues a correction. + // Yield once before doing any work, so a prompt drop/into_optimistic + // can cancel before the validator is first polled. `run_validator` is + // otherwise synchronous, so cancellation past this point is + // cooperative: it observes the cancel flag at checkpoints. tokio::task::yield_now().await; run_validator(ValidatorInput { snapshot, @@ -765,12 +862,15 @@ impl FreshnessController { now, verify_set, optimistic: optimistic_for_task, + cancelled: cancelled_for_task, + validation_block, }) }); Ok(SpeculativeSim { optimistic, validation: Some(validation), + cancelled, }) } } @@ -788,8 +888,18 @@ struct ValidatorInput { now: u64, verify_set: Vec<(Address, U256)>, optimistic: Vec, + cancelled: Arc, + /// Block the snapshot was built from; passed to the fetcher so the deferred + /// fetch reads the same block the snapshot represents. + validation_block: Option, } +/// Maximum fixed-point iterations the background validator performs while a +/// correction keeps expanding a sim's volatile read set. A backstop against +/// pathological contracts that read an unbounded chain of new volatile slots; +/// reaching it yields a best-effort `Corrected` (logged via `tracing::warn!`). +const MAX_VALIDATION_ROUNDS: u32 = 8; + /// The background validation routine. Touches only `Send` data — never the cache. fn run_validator(input: ValidatorInput) -> Validation { let ValidatorInput { @@ -804,8 +914,16 @@ fn run_validator(input: ValidatorInput) -> Validation { now, verify_set, optimistic, + cancelled, + validation_block, } = input; + // Checkpoint: cancelled before we even begin (the caller dropped or + // `into_optimistic`d the handle while we were parked at the initial yield). + if cancelled.load(Ordering::Relaxed) { + return Validation::Confirmed; + } + let Some(fetcher) = fetcher else { return Validation::Unverified { reason: "no storage batch fetcher available".to_string(), @@ -826,8 +944,14 @@ fn run_validator(input: ValidatorInput) -> Validation { } let verify: Vec<(Address, U256)> = verify.into_iter().collect(); + // Checkpoint: cancelled before issuing the (costly, side-effecting) fetch. + // This is what makes the "dropped before fetching" guarantee hold. + if cancelled.load(Ordering::Relaxed) { + return Validation::Confirmed; + } + // Fetch fresh values. Any error → Unverified (never trust silently). - let results = (fetcher)(verify.clone()); + let results = (fetcher)(verify.clone(), validation_block); let mut fresh: HashMap<(Address, U256), U256> = HashMap::new(); for (addr, slot, value) in results { match value { @@ -842,8 +966,17 @@ fn run_validator(input: ValidatorInput) -> Validation { } } - // Compare against the snapshot, observe each checked slot, collect changes. - let mut changed = Vec::new(); + // Checkpoint: cancelled after the fetch returned but before we record any + // observations or queue a correction. A cancel seen here discards the + // verdict's side effects entirely. + if cancelled.load(Ordering::Relaxed) { + return Validation::Confirmed; + } + + // Compare the initial verify set against the snapshot, observe each checked + // slot, and seed the changed set (deduped by `(address, slot)`). + let mut changed_map: HashMap<(Address, U256), SlotChange> = HashMap::new(); + let mut verified: HashSet<(Address, U256)> = verify.iter().copied().collect(); { let mut tracker = tracker.lock().unwrap_or_else(|e| e.into_inner()); for &(addr, slot) in &verify { @@ -851,49 +984,160 @@ fn run_validator(input: ValidatorInput) -> Validation { let old = snapshot.storage_value(addr, slot).unwrap_or(U256::ZERO); tracker.observe(addr, slot, new, now); if new != old { - changed.push(SlotChange { - address: addr, - slot, - old, - new, - }); + changed_map.insert( + (addr, slot), + SlotChange { + address: addr, + slot, + old, + new, + }, + ); } } } - if changed.is_empty() { + if changed_map.is_empty() { return Validation::Confirmed; } - // Queue corrections for flow-back into the cache on the next run. - { - let mut pending = pending.lock().unwrap_or_else(|e| e.into_inner()); - pending.extend(changed.iter().cloned()); - } + // Re-run affected sims to a fixed point. A correction can flip control flow + // so a re-run reads a *new* volatile slot the optimistic read set never + // touched; that slot must itself be verified, or the "corrected" result + // would still rest on stale snapshot state. Each round re-runs every sim + // whose (possibly expanded) read set intersects a changed slot — applying + // the full accumulated override set — collects newly-read volatile slots, + // fetches and diffs them, and repeats until no new volatile slot appears, + // none of the newly fetched slots differ, or the iteration cap is reached. + let mut results = optimistic; + // Per-sim current volatile read set; starts at the optimistic read set and + // expands as corrections open new branches. + let mut sim_reads = read_sets; + let mut rerun_indices: HashSet = HashSet::new(); + let mut round: u32 = 0; + loop { + let changed_keys: HashSet<(Address, U256)> = changed_map.keys().copied().collect(); + let overrides: Vec<(Address, U256, U256)> = changed_map + .values() + .map(|c| (c.address, c.slot, c.new)) + .collect(); + + // Re-run sims whose current read set intersects a changed slot, applying + // every accumulated override, and gather newly-read volatile candidates. + let mut any_rerun = false; + let mut new_candidates: HashSet<(Address, U256)> = HashSet::new(); + for (i, req) in requests.iter().enumerate() { + if !sim_reads[i].iter().any(|k| changed_keys.contains(k)) { + continue; + } + any_rerun = true; + rerun_indices.insert(i); + let mut overlay = EvmOverlay::new(Arc::clone(&snapshot), None); + for &(addr, slot, value) in &overrides { + overlay.override_slot(addr, slot, value); + } + if let Ok((result, access)) = overlay.call_raw_with_access_list_with( + req.from, + req.to, + req.calldata.clone(), + &req.tx, + ) { + results[i] = result_to_sim(result, &access.to_eip2930()); + let new_volatile: Vec<(Address, U256)> = access + .slots + .iter() + .copied() + .filter(|(a, s)| registry.is_volatile(*a, *s, now)) + .collect(); + for &key in &new_volatile { + if !verified.contains(&key) { + new_candidates.insert(key); + } + } + sim_reads[i] = new_volatile; + } + } - // Re-run only the sims whose read set intersects the changed slots. - let changed_keys: HashSet<(Address, U256)> = - changed.iter().map(|c| (c.address, c.slot)).collect(); - let overrides: Vec<(Address, U256, U256)> = - changed.iter().map(|c| (c.address, c.slot, c.new)).collect(); + // No sim read a changed slot (the change came from the predicted + // candidate set, not an actual read), or no new volatile slot surfaced: + // the current results already reflect every override, so we are done. + if !any_rerun || new_candidates.is_empty() { + break; + } + // Results already reflect every override applied so far. Stop here rather + // than expanding the verified set further when the cap is reached. + if round >= MAX_VALIDATION_ROUNDS { + tracing::warn!( + rounds = round, + "freshness validator hit fixed-point iteration cap; returning best-effort Corrected" + ); + break; + } - let mut results = optimistic; - for (i, req) in requests.iter().enumerate() { - let read_set = &read_sets[i]; - let intersects = read_set.iter().any(|k| changed_keys.contains(k)); - if !intersects { - continue; + // Checkpoint: cancelled mid-loop. Results so far reflect the applied + // overrides; do not fetch further or queue corrections. + if cancelled.load(Ordering::Relaxed) { + return Validation::Confirmed; } - rerun_count.fetch_add(1, Ordering::Relaxed); - let mut overlay = EvmOverlay::new(Arc::clone(&snapshot), None); - for &(addr, slot, value) in &overrides { - overlay.override_slot(addr, slot, value); + + // Fetch the newly-discovered candidates; any error → Unverified. + let new_vec: Vec<(Address, U256)> = new_candidates.into_iter().collect(); + let fetched = (fetcher)(new_vec.clone(), validation_block); + let mut new_fresh: HashMap<(Address, U256), U256> = HashMap::new(); + for (addr, slot, value) in fetched { + match value { + Ok(v) => { + new_fresh.insert((addr, slot), v); + } + Err(e) => { + return Validation::Unverified { + reason: format!("fetch failed for {addr}:{slot}: {e}"), + }; + } + } } - if let Ok((result, access)) = - overlay.call_raw_with_access_list(req.from, req.to, req.calldata.clone()) + + // Diff + observe the newly fetched slots, growing the changed set. + let mut grew = false; { - results[i] = result_to_sim(result, &access.to_eip2930()); + let mut tracker = tracker.lock().unwrap_or_else(|e| e.into_inner()); + for &(addr, slot) in &new_vec { + verified.insert((addr, slot)); + let new = new_fresh.get(&(addr, slot)).copied().unwrap_or(U256::ZERO); + let old = snapshot.storage_value(addr, slot).unwrap_or(U256::ZERO); + tracker.observe(addr, slot, new, now); + if new != old { + changed_map.insert( + (addr, slot), + SlotChange { + address: addr, + slot, + old, + new, + }, + ); + grew = true; + } + } } + + // The newly fetched slots were all unchanged → another round would not + // alter any result; current results are final. + if !grew { + break; + } + round += 1; + } + + // Count distinct affected sims once: a sim re-run across multiple rounds is + // still one affected sim, preserving the "once per re-executed sim" contract. + rerun_count.fetch_add(rerun_indices.len(), Ordering::Relaxed); + + // Queue every accumulated correction for flow-back into the cache next run. + let changed: Vec = changed_map.into_values().collect(); + { + let mut pending = pending.lock().unwrap_or_else(|e| e.into_inner()); + pending.extend(changed.iter().cloned()); } Validation::Corrected { results, changed } @@ -902,20 +1146,31 @@ fn run_validator(input: ValidatorInput) -> Validation { /// Build a [`CallSimulationResult`] from a non-committing execution result and /// its captured access list. `token_deltas` is empty (the optimistic path does /// not run transfer tracking); gas, logs, and return data come from the -/// execution result. `output` carries the `Success`/`Revert` payload (empty on -/// `Halt`), so a corrected view-call's new return value is observable here. +/// execution result. `status` records whether the call succeeded, reverted, or +/// halted; `output` carries the `Success`/`Revert` payload (empty on `Halt`), +/// so a corrected view-call's new return value is observable here. fn result_to_sim(result: ExecutionResult, access_list: &AccessList) -> CallSimulationResult { - let (gas_used, logs, output) = match result { + let (status, gas_used, logs, output) = match result { ExecutionResult::Success { gas_used, logs, output, .. - } => (gas_used, logs, output.into_data()), - ExecutionResult::Revert { gas_used, output } => (gas_used, Vec::new(), output), - ExecutionResult::Halt { gas_used, .. } => (gas_used, Vec::new(), Bytes::new()), + } => (SimStatus::Success, gas_used, logs, output.into_data()), + ExecutionResult::Revert { gas_used, output } => { + (SimStatus::Revert, gas_used, Vec::new(), output) + } + ExecutionResult::Halt { gas_used, reason } => ( + SimStatus::Halt { + reason: format!("{reason:?}"), + }, + gas_used, + Vec::new(), + Bytes::new(), + ), }; CallSimulationResult { + status, gas_used, token_deltas: HashMap::new(), logs, diff --git a/src/inspector.rs b/src/inspector.rs index 68ce420..4c95563 100644 --- a/src/inspector.rs +++ b/src/inspector.rs @@ -5,6 +5,18 @@ //! signature, and records each transfer. The captured transfers let callers //! compute net balance changes per token and account without re-reading storage //! after the call. +//! +//! # Parsing assumptions +//! +//! Transfers are decoded assuming the standard ERC20 event layout: +//! `from` and `to` come from the indexed topics (via [`Address::from_word`], i.e. +//! the low 20 bytes of each 32-byte topic) and `value` is read from the first 32 +//! data bytes. A non-standard or packed `Transfer` event (e.g. one that does not +//! index `from`/`to`, or packs additional fields into the data) may parse +//! incorrectly or be silently skipped. +//! +//! Balance deltas are computed symmetrically: a self-transfer where `from == to` +//! is both subtracted and added, netting to zero for that owner. use std::collections::HashMap; @@ -12,31 +24,56 @@ use alloy_primitives::{Address, B256, I256, Log, U256}; use revm::Inspector; use revm::interpreter::InterpreterTypes; -/// ERC20 Transfer event signature: keccak256("Transfer(address,address,uint256)") +/// ERC20 `Transfer` event signature: `keccak256("Transfer(address,address,uint256)")`. +/// +/// A log's first topic must equal this value to be treated as a transfer. const TRANSFER_EVENT_SIGNATURE: B256 = B256::new([ 0xdd, 0xf2, 0x52, 0xad, 0x1b, 0xe2, 0xc8, 0x9b, 0x69, 0xc2, 0xb0, 0x68, 0xfc, 0x37, 0x8d, 0xaa, 0x95, 0x2b, 0xa7, 0xf1, 0x63, 0xc4, 0xa1, 0x16, 0x28, 0xf5, 0x5a, 0x4d, 0xf5, 0x23, 0xb3, 0xef, ]); -/// Represents a single ERC20 token transfer +/// A single ERC20 token transfer decoded from a `Transfer` log. +/// +/// Fields are populated from the standard ERC20 event layout (see the +/// [module docs](crate::inspector) for caveats on non-standard events). #[derive(Clone, Debug, PartialEq, Eq)] pub struct TokenTransfer { + /// Address of the token contract that emitted the event (the log's address). pub token: Address, + /// Sender, decoded from the first indexed topic. pub from: Address, + /// Recipient, decoded from the second indexed topic. pub to: Address, + /// Amount transferred, decoded from the first 32 data bytes. pub value: U256, } -/// Inspector that captures ERC20 Transfer events during EVM execution +/// Inspector that captures ERC20 `Transfer` events during EVM execution. +/// +/// Attach to a simulation and the [`Inspector::log`] hook records every emitted +/// log; logs matching the ERC20 `Transfer` layout are additionally decoded into +/// [`TokenTransfer`]s. Reconstruct net balance changes afterward with +/// [`balance_deltas`](Self::balance_deltas) or +/// [`balance_deltas_for_tokens`](Self::balance_deltas_for_tokens), and reuse the +/// inspector across calls via [`clear`](Self::clear). #[derive(Clone, Debug, Default)] pub struct TransferInspector { - /// All captured token transfers + /// Token transfers decoded from captured logs. pub transfers: Vec, - /// All logs emitted during execution + /// Every log emitted during execution, retained for debugging/analysis. pub logs: Vec, } impl TransferInspector { + /// Create an empty inspector with no captured transfers or logs. + /// + /// ``` + /// use evm_fork_cache::inspector::TransferInspector; + /// + /// let inspector = TransferInspector::new(); + /// assert!(inspector.transfers.is_empty()); + /// assert!(inspector.logs.is_empty()); + /// ``` pub fn new() -> Self { Self { transfers: Vec::new(), @@ -67,7 +104,29 @@ impl TransferInspector { deltas } - /// Filter balance deltas to only include specified tokens + /// Like [`balance_deltas`](Self::balance_deltas), but restricted to the + /// given set of token addresses. + /// + /// Tokens in `tokens` with no transfers touching `owner` are simply absent + /// from the result; tokens not in `tokens` are excluded even if `owner` + /// transacted in them. + /// + /// ``` + /// # use evm_fork_cache::inspector::{TransferInspector, TokenTransfer}; + /// # use alloy_primitives::{Address, I256, U256}; + /// let mut inspector = TransferInspector::new(); + /// let token_a = Address::repeat_byte(0xAA); + /// let token_b = Address::repeat_byte(0xBB); + /// let owner = Address::repeat_byte(0x11); + /// let other = Address::repeat_byte(0x22); + /// inspector.transfers.push(TokenTransfer { token: token_a, from: owner, to: other, value: U256::from(100u64) }); + /// inspector.transfers.push(TokenTransfer { token: token_b, from: other, to: owner, value: U256::from(50u64) }); + /// + /// let deltas = inspector.balance_deltas_for_tokens(owner, [token_a]); + /// assert_eq!(deltas.len(), 1); + /// assert_eq!(deltas.get(&token_a), Some(&(-I256::from_raw(U256::from(100u64))))); + /// assert!(!deltas.contains_key(&token_b)); + /// ``` pub fn balance_deltas_for_tokens( &self, owner: Address, @@ -82,7 +141,8 @@ impl TransferInspector { .collect() } - /// Clear all captured data for reuse + /// Drop all captured transfers and logs so the inspector can be reused + /// across simulations. pub fn clear(&mut self) { self.transfers.clear(); self.logs.clear(); @@ -126,10 +186,18 @@ impl TransferInspector { } } +/// Captures every emitted log via the [`Inspector::log`] hook. +/// +/// Each log is pushed to [`logs`](TransferInspector::logs); logs whose first +/// topic matches the ERC20 `Transfer` signature and that carry the standard ERC20 +/// layout are additionally decoded into [`transfers`](TransferInspector::transfers). +/// Logs that do not match (wrong signature, fewer than three topics, or fewer +/// than 32 data bytes) are retained in `logs` but produce no transfer. impl Inspector for TransferInspector where INTR: InterpreterTypes, { + /// Records `log` and, if it parses as an ERC20 `Transfer`, the decoded transfer. fn log(&mut self, _context: &mut CTX, log: Log) { // Try to parse as ERC20 Transfer event if let Some(transfer) = Self::parse_transfer(&log) { diff --git a/src/lib.rs b/src/lib.rs index 1025ff0..434c451 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,35 +1,96 @@ -//! Forked EVM state cache and simulation utilities for DeFi search. -//! -//! `evm-fork-cache` is a support layer for simulating EVM transactions against -//! recent on-chain state without re-deriving it on every call. It builds on -//! `revm` and `foundry-fork-db` to provide a lazy-loading state cache, -//! immutable snapshots that can be shared across threads, per-simulation -//! overlays, and a set of helpers for the kinds of state manipulation a search -//! loop needs (overriding ERC20 balances by scanning for the balance slot, -//! batched `eth_call` multicalls, Foundry-style bytecode etching, and CREATE3 -//! address derivation). -//! -//! The entry point is [`cache::EvmCache`]: construct one over an RPC backend, -//! then snapshot it with [`cache::EvmCache::create_snapshot`] to fan out -//! parallel simulations, each driving its own [`cache::EvmOverlay`]. -//! -//! Other modules: +//! Forked EVM **simulation engine** for DeFi search, MEV, and backtesting. +//! +//! `evm-fork-cache` simulates EVM transactions against recent on-chain state +//! without re-deriving that state on every call. It builds on [`revm`], +//! [`alloy`], and [`foundry-fork-db`] to provide a lazy-loading state cache, +//! immutable snapshots shareable across threads, per-simulation overlays, a +//! freshness control plane, and the state-manipulation helpers a search loop +//! needs (balance overrides, batched multicalls, Foundry-style bytecode etching, +//! CREATE3 address derivation, and an extensible revert decoder). +//! +//! [`revm`]: https://github.com/bluealloy/revm +//! [`alloy`]: https://github.com/alloy-rs/alloy +//! [`foundry-fork-db`]: https://github.com/foundry-rs/foundry-fork-db +//! +//! # The state stack +//! +//! Reads flow up; the fork DB lazily fetches misses from RPC. Writes and purges +//! are applied directly to the cache (no RPC on the hot path). +//! +//! ```text +//! EvmOverlay × N isolated, Send simulations (cheap Arc clones) +//! ▲ clone × N +//! EvmSnapshot immutable, point-in-time, Send + Sync +//! ▲ create_snapshot() +//! EvmCache lazy RPC fetch + local state cache + targeted writes/purge +//! ▲ lazy fetch +//! RPC provider +//! ``` +//! +//! The entry point is [`cache::EvmCache`]: construct one over an RPC backend +//! (see [`cache::EvmCacheBuilder`]), then snapshot it with +//! [`cache::EvmCache::create_snapshot`] to fan out parallel simulations, each +//! driving its own [`cache::EvmOverlay`]. `EvmCache` is `!Send` (it owns the +//! mutable fork and blocks on RPC internally); `EvmSnapshot` is `Send + Sync` +//! and `EvmOverlay` is `Send`, so the fan-out parallelizes safely. +//! +//! # Modules +//! +//! - [`cache`] — the fork cache, snapshots, overlays, and on-disk persistence. //! - [`access_list`] / [`access_set`] — EIP-2930 access-list construction and -//! warm-slot tracking for gas estimation. -//! - [`errors`] — structured simulation errors and an extensible revert-reason -//! decoder you can teach your own custom Solidity error selectors. +//! EIP-2929 warm-slot tracking for gas estimation. +//! - [`errors`] — structured simulation errors ([`errors::SimError`]) and an +//! extensible revert-reason decoder you can teach your own custom Solidity +//! error selectors. //! - [`freshness`] — the four-layer freshness model (classification, observation, //! policy, mechanism) and the optimistic verify-and-rerun execution loop with //! deferred validation. -//! - [`inspector`] — an `Inspector` that captures ERC20 `Transfer` events to -//! reconstruct balance deltas from a simulation. -//! - [`multicall`] — batched read-only calls. +//! - [`inspector`] — an [`Inspector`](revm::Inspector) that captures ERC20 +//! `Transfer` events to reconstruct balance deltas from a simulation. +//! - [`multicall`] — batched read-only calls through Multicall3. //! - [`deploy`] / [`create3`] — contract deployment and CREATE3 address math. //! - [`prefetch_registry`] — two-stage storage-slot pre-warming. //! -//! The `examples/` directory has runnable, documented walkthroughs of each of -//! these — offline ones that need no network, plus a few that fork real chain +//! # Requirements +//! +//! Any constructor or method that may touch RPC fetches missing state through a +//! synchronous façade over an async provider +//! ([`tokio::task::block_in_place`]), so it must run on a **multi-thread** tokio +//! runtime: +//! +//! ```ignore +//! #[tokio::main(flavor = "multi_thread")] +//! async fn main() { /* ... */ } +//! +//! #[tokio::test(flavor = "multi_thread")] +//! async fn my_test() { /* ... */ } +//! ``` +//! +//! Running on a current-thread runtime panics when a fetch is attempted. The +//! offline examples and integration tests build the cache over a mocked provider +//! and never reach the network, so they are exempt. +//! +//! # Error handling +//! +//! Simulation entry points that distinguish failure modes return +//! [`errors::SimulationResult`] (`Result`), where +//! [`SimError`](errors::SimError) separates a decoded [`Revert`](errors::SimError::Revert), +//! an EVM [`Halt`](errors::SimError::Halt), and an unexpected host-side +//! [`Other`](errors::SimError::Other) error (RPC, database, ABI encoding). The +//! freshness loop never silently trusts stale data: a transient RPC failure +//! surfaces as [`freshness::Validation::Unverified`] so callers can retry rather +//! than act on unverified results. +//! +//! # Maturity & stability +//! +//! This crate is **pre-1.0** and developed against a phased roadmap (see +//! `docs/ROADMAP.md`). Until 1.0, breaking changes may land in minor releases; +//! each is recorded in the crate `CHANGELOG.md`. MSRV is Rust 1.88 (edition 2024). +//! +//! The `examples/` directory has runnable, documented walkthroughs of each +//! module — offline ones that need no network, plus a few that fork real chain //! state over RPC. See the crate README for the full list. +#![cfg_attr(docsrs, feature(doc_cfg))] pub mod access_list; pub mod access_set; diff --git a/src/multicall.rs b/src/multicall.rs index 8ea4d72..41a7a16 100644 --- a/src/multicall.rs +++ b/src/multicall.rs @@ -18,8 +18,10 @@ use crate::cache::EvmCache; /// Multicall3 contract address (same on all EVM chains). pub const MULTICALL3_ADDRESS: Address = address!("cA11bde05977b3631167028862bE2a173976CA11"); -/// Maximum number of calls to batch in a single multicall. -/// This prevents hitting gas limits or creating overly large calldata. +/// Maximum number of calls to batch in a single `aggregate3` invocation. +/// +/// Caps per-batch gas and calldata size. [`execute_batched`] splits larger call +/// sets into chunks of at most this many calls. pub const MAX_BATCH_SIZE: usize = 200; sol! { @@ -44,18 +46,31 @@ sol! { } } -/// A batch of calls to execute via Multicall3. +/// A batch of calls to execute in a single `aggregate3` invocation via Multicall3. +/// +/// Build a batch with [`add`](Self::add) / [`add_call`](Self::add_call), then run +/// it with [`execute`](Self::execute) or [`execute_tracked`](Self::execute_tracked). +/// The batch should hold at most [`MAX_BATCH_SIZE`] calls; use [`execute_batched`] +/// to chunk larger sets automatically. pub struct MulticallBatch { calls: Vec, } impl MulticallBatch { /// Create a new empty batch. + /// + /// ``` + /// use evm_fork_cache::multicall::MulticallBatch; + /// + /// let batch = MulticallBatch::new(); + /// assert!(batch.is_empty()); + /// assert_eq!(batch.len(), 0); + /// ``` pub fn new() -> Self { Self { calls: Vec::new() } } - /// Create a new batch with pre-allocated capacity. + /// Create a new empty batch with room for `capacity` calls before reallocating. pub fn with_capacity(capacity: usize) -> Self { Self { calls: Vec::with_capacity(capacity), @@ -76,7 +91,12 @@ impl MulticallBatch { self } - /// Add a typed call to the batch. + /// Add a typed [`SolCall`] to the batch, ABI-encoding its calldata. + /// + /// Convenience wrapper over [`add`](Self::add) for callers holding a generated + /// call type rather than raw bytes. As with `add`, `allow_failure` controls + /// whether a revert of this call fails the whole batch (`false`) or surfaces as + /// `success = false` in the result (`true`). pub fn add_call( &mut self, target: Address, @@ -86,20 +106,37 @@ impl MulticallBatch { self.add(target, call.abi_encode().into(), allow_failure) } - /// Get the number of calls in the batch. + /// Number of calls currently in the batch. pub fn len(&self) -> usize { self.calls.len() } - /// Check if the batch is empty. + /// Returns `true` if the batch contains no calls. pub fn is_empty(&self) -> bool { self.calls.is_empty() } - /// Execute the batch using the provided EvmCache. + /// Execute the batch against `cache`, returning one [`IMulticall3::Result`] + /// per input call, in order. An empty batch returns an empty vector without + /// touching the EVM. + /// + /// Per-call failure is reported in the result's `success` field rather than as + /// an `Err`: a call added with `allow_failure = true` that reverts surfaces as + /// `success = false` with whatever revert data it returned. The batch as a whole + /// is all-or-nothing — a call added with `allow_failure = false` that reverts + /// makes the entire `aggregate3` call revert, which is returned here as an `Err`. + /// + /// Requires Multicall3 to be deployed at [`MULTICALL3_ADDRESS`] on the forked + /// chain (it is on virtually all EVM chains). + /// + /// # Errors /// - /// Returns a vector of results, one for each call in the batch. - /// Failed calls (when allow_failure was true) will have `success = false`. + /// Returns an error if: + /// - the underlying `call_raw` execution does not return + /// [`ExecutionResult::Success`](revm::context::result::ExecutionResult::Success) + /// — e.g. the `aggregate3` call reverted because a call with + /// `allow_failure = false` failed, or Multicall3 is not deployed; or + /// - the returned data cannot be ABI-decoded into the expected result list. #[instrument(skip(self, cache), fields(batch_size = self.calls.len()))] pub fn execute(&self, cache: &mut EvmCache) -> Result> { if self.calls.is_empty() { @@ -132,11 +169,21 @@ impl MulticallBatch { } } - /// Execute the batch and return both results and the access list of all - /// accounts/storage slots touched during execution. + /// Execute the batch and return both the results and the + /// [`StorageAccessList`] of all accounts/storage slots touched during + /// execution. /// - /// Same as [`Self::execute`] but uses `call_raw_with_access_list` to capture - /// the EVM state touched by the multicall, enabling prefetch on the next cycle. + /// Same all-or-nothing batch semantics and Multicall3 deployment requirement + /// as [`execute`](Self::execute), but uses `call_raw_with_access_list` to + /// capture the EVM state touched by the multicall, enabling prefetch on the + /// next cycle. An empty batch returns an empty result list and a default + /// (empty) access list. + /// + /// # Errors + /// + /// Returns an error under the same conditions as [`execute`](Self::execute): + /// the `aggregate3` call did not succeed (revert, or Multicall3 not deployed), + /// or the returned data failed to ABI-decode. #[instrument(skip(self, cache), fields(batch_size = self.calls.len()))] pub fn execute_tracked( &self, @@ -183,8 +230,15 @@ impl Default for MulticallBatch { /// Execute multiple calls in batches using Multicall3. /// -/// This helper handles splitting large call sets into multiple batches -/// that respect the MAX_BATCH_SIZE limit. +/// Splits large call sets into consecutive batches of at most [`MAX_BATCH_SIZE`] +/// calls, running each via [`MulticallBatch::execute`]. Results are concatenated +/// in input order. Requires Multicall3 to be deployed at [`MULTICALL3_ADDRESS`] +/// on the forked chain. +/// +/// As with a single batch, the all-or-nothing semantics are per-batch: a call +/// added with `allow_failure = true` that reverts surfaces as `success = false` +/// in its result, whereas a call with `allow_failure = false` that reverts makes +/// that batch's `aggregate3` revert (returned here as an `Err`). /// /// # Arguments /// * `cache` - The EvmCache to execute calls on @@ -192,6 +246,13 @@ impl Default for MulticallBatch { /// /// # Returns /// A vector of results in the same order as the input calls. +/// +/// # Errors +/// +/// Returns an error as soon as any chunk's [`MulticallBatch::execute`] fails — +/// i.e. that chunk's `aggregate3` reverted (a `allow_failure = false` call failed, +/// or Multicall3 is not deployed) or its return data failed to decode. Results +/// from earlier successful chunks are discarded. #[instrument(skip(cache, calls))] pub fn execute_batched(cache: &mut EvmCache, calls: I) -> Result> where @@ -225,7 +286,12 @@ where Ok(all_results) } -/// Decode a multicall result into the expected return type. +/// Decode a single multicall [`IMulticall3::Result`] into the call's typed return. +/// +/// # Errors +/// +/// Returns an error if `result.success` is `false` (the call reverted), or if +/// `result.returnData` cannot be ABI-decoded into `C::Return`. pub fn decode_result(result: &IMulticall3::Result) -> Result { if !result.success { return Err(anyhow!("Call failed")); @@ -235,7 +301,8 @@ pub fn decode_result(result: &IMulticall3::Result) -> Result(result: &IMulticall3::Result) -> Option { if !result.success { return None; diff --git a/src/prefetch_registry.rs b/src/prefetch_registry.rs index f4fb66c..ba5ab86 100644 --- a/src/prefetch_registry.rs +++ b/src/prefetch_registry.rs @@ -33,7 +33,13 @@ pub struct PrefetchRegistry { } impl PrefetchRegistry { - /// Load from disk (bincode format). Returns empty registry if file missing or corrupt. + /// Load a registry from `path` (bincode format). + /// + /// Returns [`Default`] (an empty registry) on any error — a missing file, an + /// unreadable file, or corrupt/undecodable contents. These cases are not + /// distinguished by the return value: a corrupt registry is indistinguishable + /// from a fresh start, so a decode failure silently discards previously + /// persisted prefetch data (logged at `warn`). pub fn load(path: &Path) -> Self { match std::fs::read(path) { Ok(data) => match bincode::deserialize::(&data) { @@ -71,7 +77,14 @@ impl PrefetchRegistry { } } - /// Persist to disk (bincode format). + /// Persist the registry to `path` in bincode format, creating parent + /// directories as needed. + /// + /// This is best-effort: I/O and serialization failures (unwritable parent + /// directory, failed write, or a serialization error) are logged at `warn` + /// and swallowed rather than returned, so a save failure leaves stale or + /// missing on-disk data that [`load`](Self::load) will silently treat as an + /// empty registry on the next cycle. pub fn save(&self, path: &Path) { if let Some(parent) = path.parent() && let Err(e) = std::fs::create_dir_all(parent) @@ -99,12 +112,44 @@ impl PrefetchRegistry { } } - /// Record an aggregated access list for a phase (replaces any existing). + /// Record the aggregated access list for `phase`, **overwriting** any access + /// list previously recorded for that phase. + /// + /// Each call wholesale replaces the phase's slot set; it does not merge with + /// the prior list. To accumulate per-address lists instead, use + /// [`record_keyed`](Self::record_keyed). + /// + /// ``` + /// use evm_fork_cache::prefetch_registry::PrefetchRegistry; + /// use evm_fork_cache::StorageAccessList; + /// use alloy_primitives::{Address, U256}; + /// + /// let mut registry = PrefetchRegistry::default(); + /// let addr = Address::repeat_byte(0x01); + /// + /// let mut al = StorageAccessList::default(); + /// al.slots.insert((addr, U256::from(1))); + /// registry.record("pool_refresh", al); + /// assert!(registry.phase_slots("pool_refresh").contains(&(addr, U256::from(1)))); + /// + /// // A second record replaces the slot set rather than merging. + /// let mut al2 = StorageAccessList::default(); + /// al2.slots.insert((addr, U256::from(2))); + /// registry.record("pool_refresh", al2); + /// let slots = registry.phase_slots("pool_refresh"); + /// assert_eq!(slots.len(), 1); + /// assert!(slots.contains(&(addr, U256::from(2)))); + /// ``` pub fn record(&mut self, phase: &str, access_list: StorageAccessList) { self.phases.insert(phase.to_string(), access_list); } - /// Record a keyed access list within a phase. + /// Record the access list for a single `key` within a keyed `phase`. + /// + /// Unlike [`record`](Self::record), this **inserts into** the phase's per-key + /// nested map: other keys already recorded under `phase` are preserved, and + /// only the entry for `key` is replaced. Pairs with + /// [`prefetch_keyed`](Self::prefetch_keyed). pub fn record_keyed(&mut self, phase: &str, key: Address, access_list: StorageAccessList) { self.keyed_phases .entry(phase.to_string()) @@ -166,8 +211,12 @@ impl PrefetchRegistry { batch_prefetch(cache, slots.into_iter(), phase) } - /// Returns the set of (address, slot) pairs for an aggregated phase. - /// Used to build exclusion sets for subsequent prefetches. + /// Returns the set of `(address, slot)` pairs recorded for an aggregated + /// `phase`, or an empty set if the phase was never [`record`](Self::record)ed. + /// + /// Typically used to build the `exclude` set passed to + /// [`prefetch_keyed`](Self::prefetch_keyed) so a later stage skips slots a + /// prior aggregated prefetch already warmed. pub fn phase_slots(&self, phase: &str) -> HashSet<(Address, U256)> { self.phases .get(phase) @@ -176,7 +225,15 @@ impl PrefetchRegistry { } } -/// Batch-fetch slots into the EVM cache via `storage_batch_fetcher`. +/// Batch-fetch `slots` into `cache` via its `storage_batch_fetcher` and inject +/// the results, returning `(fetched, errors)`. +/// +/// Deduplicating, exclusion, and phase lookup are the caller's responsibility +/// ([`PrefetchRegistry::prefetch_phase`] / [`PrefetchRegistry::prefetch_keyed`]). +/// If `slots` is empty, or the cache has no batch fetcher configured, returns +/// `(0, 0)` without fetching. Otherwise each slot that the fetcher resolves +/// successfully is injected into the cache and counted in `fetched`; per-slot +/// fetch errors are counted in `errors` and skipped. fn batch_prefetch( cache: &mut EvmCache, slots: impl Iterator, @@ -200,7 +257,8 @@ fn batch_prefetch( let start = std::time::Instant::now(); let total_requested = requests.len(); - let results = fetcher(requests); + // `None`: fetch at the cache's currently-pinned block (synchronous, no repin race). + let results = fetcher(requests, None); let mut successes: Vec<(Address, U256, U256)> = Vec::with_capacity(results.len()); let mut errors = 0usize; diff --git a/tests/common/mod.rs b/tests/common/mod.rs index dbdadfb..509a1f5 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -8,6 +8,7 @@ use std::collections::HashMap; use std::sync::Arc; +use alloy_eips::BlockId; use alloy_primitives::{Address, Bytes, U256, hex}; use alloy_provider::RootProvider; use alloy_provider::network::AnyNetwork; @@ -102,22 +103,24 @@ pub fn balance_of(cache: &mut EvmCache, token: Address, owner: Address) -> Resul /// how an unseen slot reads in a simulation). This is the offline stand-in for /// the real RPC batch fetcher. pub fn stub_fetcher(values: HashMap<(Address, U256), U256>) -> StorageBatchFetchFn { - Arc::new(move |requests: Vec<(Address, U256)>| { - requests - .into_iter() - .map(|(addr, slot)| { - let value = values.get(&(addr, slot)).copied().unwrap_or(U256::ZERO); - (addr, slot, Ok(value)) - }) - .collect() - }) + Arc::new( + move |requests: Vec<(Address, U256)>, _block: Option| { + requests + .into_iter() + .map(|(addr, slot)| { + let value = values.get(&(addr, slot)).copied().unwrap_or(U256::ZERO); + (addr, slot, Ok(value)) + }) + .collect() + }, + ) } /// Build a stub [`StorageBatchFetchFn`] that fails every request. /// /// Used to exercise the `Unverified` / error paths offline. pub fn failing_fetcher() -> StorageBatchFetchFn { - Arc::new(|requests: Vec<(Address, U256)>| { + Arc::new(|requests: Vec<(Address, U256)>, _block: Option| { requests .into_iter() .map(|(addr, slot)| (addr, slot, Err(anyhow!("stub fetcher error")))) @@ -135,23 +138,27 @@ pub fn tracking_fetcher( values: HashMap<(Address, U256), U256>, called: Arc, ) -> StorageBatchFetchFn { - Arc::new(move |requests: Vec<(Address, U256)>| { - called.store(true, std::sync::atomic::Ordering::SeqCst); - requests - .into_iter() - .map(|(addr, slot)| { - let value = values.get(&(addr, slot)).copied().unwrap_or(U256::ZERO); - (addr, slot, Ok(value)) - }) - .collect() - }) + Arc::new( + move |requests: Vec<(Address, U256)>, _block: Option| { + called.store(true, std::sync::atomic::Ordering::SeqCst); + requests + .into_iter() + .map(|(addr, slot)| { + let value = values.get(&(addr, slot)).copied().unwrap_or(U256::ZERO); + (addr, slot, Ok(value)) + }) + .collect() + }, + ) } /// Build a stub [`StorageBatchFetchFn`] that panics, to exercise the validator's /// `JoinError` (`Unverified`) path. pub fn panicking_fetcher() -> StorageBatchFetchFn { Arc::new( - |_requests: Vec<(Address, U256)>| -> Vec<(Address, U256, Result)> { + |_requests: Vec<(Address, U256)>, + _block: Option| + -> Vec<(Address, U256, Result)> { panic!("panicking fetcher: deliberate failure for the Unverified test") }, ) diff --git a/tests/errors.rs b/tests/errors.rs new file mode 100644 index 0000000..a30dd5c --- /dev/null +++ b/tests/errors.rs @@ -0,0 +1,135 @@ +//! Integration tests for the public error-handling surface. +//! +//! The inline unit tests in `src/errors.rs` cover revert *decoding*; these cover +//! the public `SimError` ergonomics a caller branches on (classification, +//! `Display`, `From` conversions), the decoder's `Send + Sync + Clone` sharing +//! across threads, and two edge cases flagged in `docs/KNOWN_ISSUES.md`: silent +//! selector shadowing and out-of-range panic codes. + +use std::sync::Arc; + +use alloy_primitives::{Bytes, U256}; +use alloy_sol_types::{Panic, SolError, sol}; +use anyhow::anyhow; +use evm_fork_cache::errors::{ + PANIC_SELECTOR, RevertDecoder, RevertReason, SimError, SimulationError, +}; + +sol! { + #[derive(Debug)] + error Unauthorized(address caller); +} + +#[test] +fn sim_error_classification() { + let revert: SimError = SimulationError::from_revert(21_000, Bytes::new()).into(); + assert!(revert.is_revert()); + assert!(!revert.is_halt()); + assert!(revert.as_revert().is_some()); + + let halt = SimError::Halt { + reason: "OutOfGas".to_string(), + gas_used: 1_000_000, + }; + assert!(halt.is_halt()); + assert!(!halt.is_revert()); + assert!(halt.as_revert().is_none()); + + let other: SimError = anyhow!("rpc exploded").into(); + assert!(!other.is_revert()); + assert!(!other.is_halt()); + assert!(other.as_revert().is_none()); +} + +#[test] +fn sim_error_display_distinguishes_variants() { + let revert: SimError = SimulationError::from_revert(0, Bytes::new()).into(); + assert!(revert.to_string().contains("reverted"), "{revert}"); + + let halt = SimError::Halt { + reason: "StackOverflow".to_string(), + gas_used: 5, + }; + let shown = halt.to_string(); + assert!(shown.contains("halted"), "{shown}"); + assert!(shown.contains("StackOverflow"), "{shown}"); + + let other: SimError = anyhow!("boom").into(); + assert_eq!(other.to_string(), "boom"); +} + +#[test] +fn decoder_is_shareable_across_threads() { + // A configured decoder is Send + Sync + Clone, so it can back parallel sims. + let decoder = Arc::new(RevertDecoder::new().with_error::()); + let data = Bytes::from( + Unauthorized { + caller: alloy_primitives::Address::repeat_byte(0xAB), + } + .abi_encode(), + ); + + let handles: Vec<_> = (0..4) + .map(|_| { + let decoder = Arc::clone(&decoder); + let data = data.clone(); + std::thread::spawn(move || matches!(decoder.decode(&data), RevertReason::Custom(_))) + }) + .collect(); + + for handle in handles { + assert!(handle.join().expect("thread panicked")); + } +} + +#[test] +fn duplicate_selector_registration_shadows_silently() { + // Pin the documented shadowing behavior (KNOWN_ISSUES): re-registering a + // selector replaces the prior decoder with no error, keeping len at 1. + let mut decoder = RevertDecoder::new(); + decoder.register_raw([0x11, 0x22, 0x33, 0x44], "First(uint256)", |_| { + Some("first".to_string()) + }); + decoder.register_raw([0x11, 0x22, 0x33, 0x44], "Second(uint256)", |_| { + Some("second".to_string()) + }); + assert_eq!(decoder.len(), 1, "second registration replaced the first"); + + let data = Bytes::from(vec![0x11, 0x22, 0x33, 0x44]); + match decoder.decode(&data) { + RevertReason::Custom(custom) => { + assert_eq!(custom.name, "Second(uint256)"); + assert_eq!(custom.params.as_deref(), Some("second")); + } + other => panic!("expected the shadowing Custom error, got {other}"), + } +} + +#[test] +fn out_of_range_panic_code_falls_through_to_unknown() { + // A Panic(uint256) whose code exceeds u64::MAX cannot be represented, so it + // is reported as Unknown rather than Panic (KNOWN_ISSUES item 7). + let data = Bytes::from(Panic { code: U256::MAX }.abi_encode()); + match RevertDecoder::new().decode(&data) { + RevertReason::Unknown { selector, .. } => { + assert_eq!(selector.as_slice(), &PANIC_SELECTOR); + } + other => panic!("expected Unknown for an out-of-range panic, got {other}"), + } +} + +#[test] +fn in_range_panic_code_decodes_to_panic() { + // The companion to the overflow case: a normal single-byte panic code + // decodes to Panic, confirming the selector itself is wired up correctly. + let data = Bytes::from( + Panic { + code: U256::from(0x11u64), + } + .abi_encode(), + ); + assert_eq!( + RevertDecoder::new().decode(&data), + RevertReason::Panic(0x11) + ); +} diff --git a/tests/freshness.rs b/tests/freshness.rs index 9eb36d1..a892883 100644 --- a/tests/freshness.rs +++ b/tests/freshness.rs @@ -10,6 +10,7 @@ mod common; use std::collections::HashMap; use std::sync::{Arc, Mutex}; +use alloy_eips::BlockId; use alloy_primitives::{Address, Bytes, U256}; use alloy_sol_types::SolCall; use anyhow::Result; @@ -18,7 +19,9 @@ use common::{ MOCK_ERC20_BALANCE_SLOT, MockERC20, failing_fetcher, install_default_account, install_mock_erc20, panicking_fetcher, setup_cache, stub_fetcher, tracking_fetcher, }; -use evm_fork_cache::cache::{EvmCache, EvmOverlay, SlotObservationTracker}; +use evm_fork_cache::cache::{ + EvmCache, EvmOverlay, SimStatus, SlotObservationTracker, StorageBatchFetchFn, +}; use evm_fork_cache::freshness::{ AlwaysVerify, BlockClock, FreshnessController, FreshnessParams, FreshnessRegistry, NeverVerify, ObservationDriven, SimRequest, Validation, WallClock, @@ -601,6 +604,506 @@ async fn run_drains_pending_on_next_run() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread")] +async fn run_converges_when_corrected_slot_is_overlay_resident() -> Result<()> { + // F1 regression: a correction must reach the layer that *wins* in the + // snapshot. When the verified slot lives in the CacheDB overlay (layer 1) — + // e.g. seeded via insert_account_storage or written by a committed call — + // draining the correction into BlockchainDb (layer 2) alone leaves the stale + // overlay value shadowing it, so the cache never converges and re-corrects + // forever. This test seeds the balance into the overlay and asserts the + // second run both heals the live cache and yields Confirmed. + let token = Address::repeat_byte(0x11); + let owner = Address::repeat_byte(0x22); + let recipient = Address::repeat_byte(0x33); + + let mut cache = setup_cache().await?; + install_default_account(&mut cache, Address::ZERO); + install_default_account(&mut cache, owner); + install_mock_erc20(&mut cache, token); + + let slot = balance_slot_for(owner); + // Seed the balance into the OVERLAY (layer 1), not layer 2. + cache + .db_mut() + .insert_account_storage(token, slot, U256::from(1000))?; + assert_eq!( + cache.cached_storage_value(token, slot), + Some(U256::from(1000)), + "precondition: overlay holds the seeded value" + ); + + // Live value is 2000 (changed). + cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([( + (token, slot), + U256::from(2000), + )]))); + + let mut controller = FreshnessController::new(FreshnessRegistry::new(), AlwaysVerify); + + // First run: detects the change, queues a correction. + let sim = controller.run( + &mut cache, + vec![SimRequest::new( + owner, + token, + transfer_calldata(recipient, U256::from(100)), + )], + )?; + assert!(matches!(sim.validate().await, Validation::Corrected { .. })); + assert_eq!(controller.pending_len(), 1); + + // Second run: drains the correction. It must overwrite the overlay-resident + // slot, not just layer 2, so the live cache now reads the fresh value. + let sim = controller.run( + &mut cache, + vec![SimRequest::new( + owner, + token, + transfer_calldata(recipient, U256::from(100)), + )], + )?; + assert_eq!(controller.pending_len(), 0, "pending drained"); + assert_eq!( + cache.cached_storage_value(token, slot), + Some(U256::from(2000)), + "correction must overwrite the overlay-resident slot, not just layer 2" + ); + + // The snapshot now matches the fetcher → Confirmed, proving convergence. + let validation = sim.validate().await; + assert!( + matches!(validation, Validation::Confirmed), + "must converge, got {validation:?}" + ); + // No background re-run happened on the converged second cycle. + assert_eq!(controller.rerun_count(), 1, "only the first cycle re-ran"); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn verify_slots_heals_overlay_resident_slot() -> Result<()> { + // F1 regression on the synchronous primitive: verify_slots must heal a slot + // that lives in the CacheDB overlay, so both cached_storage_value and the + // EVM SLOAD path (here, a balanceOf call against a StorageCleared account) + // reflect the fresh value, and a re-verify is idempotent. + let token = Address::repeat_byte(0x11); + let owner = Address::repeat_byte(0x22); + + let mut cache = setup_cache().await?; + install_default_account(&mut cache, Address::ZERO); // coinbase, for call_raw + install_default_account(&mut cache, owner); + install_mock_erc20(&mut cache, token); + + let slot = balance_slot_for(owner); + cache + .db_mut() + .insert_account_storage(token, slot, U256::from(100))?; + + cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([( + (token, slot), + U256::from(999), + )]))); + + let changed = cache.verify_slots(&[(token, slot)])?; + assert_eq!(changed.len(), 1, "stale overlay slot detected as changed"); + assert_eq!( + cache.cached_storage_value(token, slot), + Some(U256::from(999)), + "verify_slots heals the overlay-resident slot" + ); + + // The synchronous EVM SLOAD path sees the fresh value too: the + // StorageCleared overlay account must read the written slot (a value the + // delete-the-slot alternative would have turned into a zero read). + let balance = common::balance_of(&mut cache, token, owner)?; + assert_eq!( + balance, + U256::from(999), + "EVM SLOAD reflects the healed overlay slot" + ); + + // Converged: a re-verify reports nothing (no perpetual re-change). + assert!( + cache.verify_slots(&[(token, slot)])?.is_empty(), + "overlay slot healed; re-verify is idempotent" + ); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn optimistic_result_reports_status_per_outcome() -> Result<()> { + // F6 regression: CallSimulationResult must distinguish Success from Revert + // via an explicit status. The old example inferred success from + // `!logs.is_empty()`, which misclassifies a zero-log success (a view call) + // as a revert. + let token = Address::repeat_byte(0x11); + let owner = Address::repeat_byte(0x22); + let recipient = Address::repeat_byte(0x33); + + let mut cache = setup_cache().await?; + install_default_account(&mut cache, Address::ZERO); + install_default_account(&mut cache, owner); + install_mock_erc20(&mut cache, token); + let slot = balance_slot_for(owner); + cache.inject_storage_batch(&[(token, slot, U256::from(1000))]); + cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([( + (token, slot), + U256::from(1000), + )]))); + + let mut controller = FreshnessController::new(FreshnessRegistry::new(), AlwaysVerify); + + // A balanceOf view call SUCCEEDS but emits NO logs — status must be Success, + // not the revert the old logs heuristic would have inferred. + let sim = controller.run( + &mut cache, + vec![SimRequest::new(owner, token, balance_of_calldata(owner))], + )?; + assert_eq!(sim.optimistic()[0].status, SimStatus::Success); + assert!( + sim.optimistic()[0].logs.is_empty(), + "the view call emits no logs" + ); + assert_eq!( + decode_balance(&sim.optimistic()[0].output), + U256::from(1000) + ); + sim.into_optimistic(); + + // Transferring more than the balance REVERTS — status must be Revert. + let sim = controller.run( + &mut cache, + vec![SimRequest::new( + owner, + token, + transfer_calldata(recipient, U256::from(5000)), + )], + )?; + assert_eq!(sim.optimistic()[0].status, SimStatus::Revert); + sim.into_optimistic(); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn dropping_after_fetch_started_suppresses_correction() -> Result<()> { + // F4: cancellation is best-effort, but once observed at a checkpoint it must + // prevent side effects. The fetcher is held inside a barrier so the validator + // is provably past `yield_now` and blocked mid-fetch; we drop the sim while it + // is blocked, then release it. The post-fetch checkpoint must see the cancel + // and NOT queue a correction, even though the balance slot changed. + let token = Address::repeat_byte(0x11); + let owner = Address::repeat_byte(0x22); + let recipient = Address::repeat_byte(0x33); + + let mut cache = setup_cache().await?; + install_default_account(&mut cache, Address::ZERO); + install_default_account(&mut cache, owner); + install_mock_erc20(&mut cache, token); + let slot = balance_slot_for(owner); + cache.inject_storage_batch(&[(token, slot, U256::from(1000))]); + + // Two rendezvous: R1 = "fetch started", R2 = "released by the test". After R2 + // the fetcher reports a CHANGED balance, so absent the cancel the validator + // would queue a correction. + let barrier = Arc::new(std::sync::Barrier::new(2)); + let fb = Arc::clone(&barrier); + let fetcher: StorageBatchFetchFn = + Arc::new(move |reqs: Vec<(Address, U256)>, _block: Option| { + fb.wait(); // R1 + fb.wait(); // R2 + reqs.into_iter() + .map(|(a, s)| (a, s, Ok(U256::from(2000)))) + .collect() + }); + cache.set_storage_batch_fetcher(fetcher); + + let mut controller = FreshnessController::new(FreshnessRegistry::new(), AlwaysVerify); + let sim = controller.run( + &mut cache, + vec![SimRequest::new( + owner, + token, + transfer_calldata(recipient, U256::from(100)), + )], + )?; + + barrier.wait(); // R1: the validator is now blocked inside the fetcher. + drop(sim); // Sets the cancel flag (abort cannot preempt the sync validator). + barrier.wait(); // R2: release the fetcher; the validator resumes past the fetch. + + settle().await; + assert_eq!( + controller.pending_len(), + 0, + "a cancel observed after the fetch must suppress the queued correction" + ); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn run_corrected_rerun_verifies_newly_read_volatile_slot() -> Result<()> { + // F2 regression: a correction can flip control flow so the re-run reads a + // NEW volatile slot the optimistic run never touched. That slot must itself + // be fetched and diffed (fixed-point), or the "corrected" result would still + // rest on stale snapshot state. + use revm::state::{AccountInfo, Bytecode}; + + let mut cache = setup_cache().await?; + install_default_account(&mut cache, Address::ZERO); + let caller = Address::repeat_byte(0x66); + install_default_account(&mut cache, caller); + + // Branchy runtime: load slot 0 (A); if A != 0 return A (reads only slot 0); + // else read slot 1 (B) and return it. A correction A: nonzero -> 0 flips the + // branch onto slot B, which the optimistic run never read. + let contract = Address::repeat_byte(0x55); + let code = Bytecode::new_raw(Bytes::from( + alloy_primitives::hex::decode("600054806013575060015460005260206000f35b60005260206000f3") + .expect("valid runtime hex"), + )); + let code_hash = code.hash_slow(); + cache.db_mut().insert_account_info( + contract, + AccountInfo { + balance: U256::ZERO, + nonce: 0, + code: Some(code), + code_hash, + account_id: None, + }, + ); + cache + .db_mut() + .replace_account_storage(contract, Default::default()) + .unwrap(); + + let slot_a = U256::from(0); + let slot_b = U256::from(1); + // Snapshot: A = 5 (nonzero) → optimistic takes "return A" and never reads B. + cache.inject_storage_batch(&[(contract, slot_a, U256::from(5))]); + // Fresh chain: A dropped to 0 (flips the branch) and B is 777. + cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([ + ((contract, slot_a), U256::from(0)), + ((contract, slot_b), U256::from(777)), + ]))); + + let mut controller = FreshnessController::new(FreshnessRegistry::new(), AlwaysVerify); + let sim = controller.run( + &mut cache, + vec![SimRequest::new(caller, contract, Bytes::new())], + )?; + + // Optimistic: A != 0 branch returns 5. + assert_eq!( + U256::from_be_slice(&sim.optimistic()[0].output), + U256::from(5) + ); + + match sim.validate().await { + Validation::Corrected { results, changed } => { + let keys: std::collections::HashSet<(Address, U256)> = + changed.iter().map(|c| (c.address, c.slot)).collect(); + assert!(keys.contains(&(contract, slot_a)), "A reported as changed"); + assert!( + keys.contains(&(contract, slot_b)), + "B (read only on the corrected branch) must be verified and reported" + ); + assert_eq!( + U256::from_be_slice(&results[0].output), + U256::from(777), + "corrected result must use the FRESH value of the newly-read slot, not stale 0" + ); + } + other => panic!("expected Corrected, got {other:?}"), + } + // The one affected sim, re-run across multiple rounds, is counted once. + assert_eq!(controller.rerun_count(), 1); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn overlay_call_with_tx_config_threads_value() -> Result<()> { + // F3 regression: the overlay must honor TxConfig.value, not hardcode zero. + use evm_fork_cache::cache::TxConfig; + use revm::context::result::ExecutionResult; + use revm::state::{AccountInfo, Bytecode}; + + let mut cache = setup_cache().await?; + install_default_account(&mut cache, Address::ZERO); + let caller = Address::repeat_byte(0x66); + install_default_account(&mut cache, caller); + + // Runtime bytecode that returns msg.value: + // CALLVALUE; PUSH1 0; MSTORE; PUSH1 32; PUSH1 0; RETURN. + let callee = Address::repeat_byte(0x55); + let code = Bytecode::new_raw(Bytes::from( + alloy_primitives::hex::decode("3460005260206000f3").expect("valid runtime hex"), + )); + let code_hash = code.hash_slow(); + cache.db_mut().insert_account_info( + callee, + AccountInfo { + balance: U256::ZERO, + nonce: 0, + code: Some(code), + code_hash, + account_id: None, + }, + ); + cache + .db_mut() + .replace_account_storage(callee, Default::default()) + .unwrap(); + + let snapshot = cache.create_snapshot(); + let mut overlay = EvmOverlay::new(snapshot, None); + + fn returned_value(res: ExecutionResult) -> U256 { + match res { + ExecutionResult::Success { output, .. } => U256::from_be_slice(&output.into_data()), + other => panic!("expected success, got {other:?}"), + } + } + + // The zero-value shorthand observes value 0. + let (res, _) = overlay.call_raw_with_access_list(caller, callee, Bytes::new())?; + assert_eq!(returned_value(res), U256::ZERO); + + // The TxConfig variant threads the native value through to CALLVALUE. + let tx = TxConfig { + value: U256::from(12_345u64), + ..Default::default() + }; + let (res, _) = overlay.call_raw_with_access_list_with(caller, callee, Bytes::new(), &tx)?; + assert_eq!(returned_value(res), U256::from(12_345u64)); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn run_honors_tx_gas_limit() -> Result<()> { + // F3 regression: SimRequest.tx.gas_limit must reach the optimistic call. A + // limit well below the ~51k an ERC20 transfer needs (but above intrinsic gas) + // halts out-of-gas; ignoring it would run at the default limit → Success. + let token = Address::repeat_byte(0x11); + let owner = Address::repeat_byte(0x22); + let recipient = Address::repeat_byte(0x33); + + let mut cache = setup_cache().await?; + install_default_account(&mut cache, Address::ZERO); + install_default_account(&mut cache, owner); + install_mock_erc20(&mut cache, token); + let slot = balance_slot_for(owner); + cache.inject_storage_batch(&[(token, slot, U256::from(1000))]); + cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([( + (token, slot), + U256::from(1000), + )]))); + + let mut controller = FreshnessController::new(FreshnessRegistry::new(), AlwaysVerify); + + let req = SimRequest::new(owner, token, transfer_calldata(recipient, U256::from(100))) + .with_gas_limit(30_000); + let sim = controller.run(&mut cache, vec![req])?; + assert!( + matches!(sim.optimistic()[0].status, SimStatus::Halt { .. }), + "gas-bounded transfer must halt, got {:?}", + sim.optimistic()[0].status + ); + sim.into_optimistic(); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn validator_fetches_at_snapshot_block_despite_repin() -> Result<()> { + // F5 regression: the deferred validator must fetch at the block its snapshot + // was built from, even if the cache is re-pinned while validation is pending. + // Otherwise it would compare snapshot(N) values against fresh(N+1) values and + // emit a spurious Corrected. + use alloy_eips::BlockNumberOrTag; + + let token = Address::repeat_byte(0x11); + let owner = Address::repeat_byte(0x22); + let recipient = Address::repeat_byte(0x33); + let slot = balance_slot_for(owner); + let n = 100u64; + let block_n = BlockId::Number(BlockNumberOrTag::Number(n)); + + let mut cache = setup_cache().await?; + install_default_account(&mut cache, Address::ZERO); + install_default_account(&mut cache, owner); + install_mock_erc20(&mut cache, token); + cache.set_block(Some(block_n)); + cache.inject_storage_batch(&[(token, slot, U256::from(1000))]); + assert_eq!( + cache.cached_storage_value(token, slot), + Some(U256::from(1000)), + "PRECONDITION: seeded balance present after set_block + inject" + ); + + // Block-aware fetcher: the snapshot value (1000) at block N, a CHANGED value + // (2000) at any other block. Records the block it was asked for, and blocks on + // a barrier so the test can repin before the fetch resolves. + let barrier = Arc::new(std::sync::Barrier::new(2)); + let fb = Arc::clone(&barrier); + let seen_block: Arc>>> = Arc::new(Mutex::new(None)); + let seen = Arc::clone(&seen_block); + let fetcher: StorageBatchFetchFn = + Arc::new(move |reqs: Vec<(Address, U256)>, block: Option| { + *seen.lock().unwrap() = Some(block); + fb.wait(); // R1: fetch entered + fb.wait(); // R2: released after the test repins + let at_n = block == Some(block_n); + reqs.into_iter() + .map(|(a, s)| { + // At block N every slot matches the snapshot (sender = 1000, + // everything else = 0) → Confirmed. At any other block the + // sender balance reads as changed (2000) → would be Corrected. + let v = if s == slot { + if at_n { + U256::from(1000) + } else { + U256::from(2000) + } + } else { + U256::ZERO + }; + (a, s, Ok(v)) + }) + .collect() + }); + cache.set_storage_batch_fetcher(fetcher); + + let mut controller = FreshnessController::new(FreshnessRegistry::new(), AlwaysVerify); + let sim = controller.run( + &mut cache, + vec![SimRequest::new( + owner, + token, + transfer_calldata(recipient, U256::from(100)), + )], + )?; + + barrier.wait(); // R1: the validator is inside the fetcher. + // Re-pin the cache to N+1 while validation is still outstanding. + cache.set_block(Some(BlockId::Number(BlockNumberOrTag::Number(n + 1)))); + barrier.wait(); // R2: release the fetcher. + + let verdict = sim.validate().await; + assert!( + matches!(verdict, Validation::Confirmed), + "validator must fetch at the snapshot's block N, not the re-pinned N+1; got {verdict:?}" + ); + assert_eq!( + *seen_block.lock().unwrap(), + Some(Some(block_n)), + "the fetch must be pinned to the snapshot block N" + ); + Ok(()) +} + #[tokio::test(flavor = "multi_thread")] async fn run_unverified_on_fetcher_error() -> Result<()> { let token = Address::repeat_byte(0x44); diff --git a/tests/multicall.rs b/tests/multicall.rs new file mode 100644 index 0000000..8ac1895 --- /dev/null +++ b/tests/multicall.rs @@ -0,0 +1,97 @@ +//! Offline integration tests for the Multicall3 helpers. +//! +//! The live `aggregate3` execution path requires the Multicall3 contract to be +//! deployed in the fork, which the RPC-gated `multicall_batch` example exercises. +//! These tests pin the network-free behavior: empty-batch short-circuits, the +//! result-decoding helpers, and the documented batch constants. + +mod common; + +use alloy_primitives::{Address, Bytes, U256}; +use alloy_sol_types::{SolCall, SolValue, sol}; +use anyhow::Result; + +use common::setup_cache; +use evm_fork_cache::multicall::{ + IMulticall3, MAX_BATCH_SIZE, MulticallBatch, decode_result, execute_batched, try_decode_result, +}; + +sol! { + function getValue() external returns (uint256); +} + +/// An empty batch returns empty results without invoking the EVM, on all three +/// entry points. +#[tokio::test(flavor = "multi_thread")] +async fn empty_batch_short_circuits() -> Result<()> { + let mut cache = setup_cache().await?; + + let batch = MulticallBatch::new(); + assert!(batch.is_empty()); + assert!(batch.execute(&mut cache)?.is_empty()); + + let (results, access) = batch.execute_tracked(&mut cache)?; + assert!(results.is_empty()); + assert!(access.slots.is_empty() && access.accounts.is_empty()); + + let batched = execute_batched(&mut cache, std::iter::empty::<(Address, Bytes, bool)>())?; + assert!(batched.is_empty()); + + Ok(()) +} + +/// `add` and `add_call` both append a call; length tracks the call count. +#[test] +fn batch_len_tracks_added_calls() { + let target = Address::repeat_byte(0x11); + let mut batch = MulticallBatch::with_capacity(2); + assert_eq!(batch.len(), 0); + + batch.add(target, getValueCall {}.abi_encode().into(), true); + batch.add_call(target, getValueCall {}, false); + assert_eq!(batch.len(), 2); + assert!(!batch.is_empty()); +} + +/// `decode_result` returns the typed value for a successful result and errors on +/// a failed one; `try_decode_result` mirrors this with `Option`. +#[test] +fn decode_result_honors_success_flag() { + let ok = IMulticall3::Result { + success: true, + returnData: U256::from(42u64).abi_encode().into(), + }; + let decoded = decode_result::(&ok).expect("successful result decodes"); + assert_eq!(decoded, U256::from(42u64)); + assert_eq!( + try_decode_result::(&ok), + Some(U256::from(42u64)) + ); + + let failed = IMulticall3::Result { + success: false, + returnData: Bytes::new(), + }; + assert!( + decode_result::(&failed).is_err(), + "a failed call cannot be decoded" + ); + assert_eq!(try_decode_result::(&failed), None); +} + +/// A successful result whose payload is undecodable errors (and yields `None`), +/// distinct from the `success == false` case. +#[test] +fn decode_result_rejects_garbage_payload() { + let garbage = IMulticall3::Result { + success: true, + returnData: Bytes::from_static(&[0x01, 0x02, 0x03]), + }; + assert!(decode_result::(&garbage).is_err()); + assert_eq!(try_decode_result::(&garbage), None); +} + +#[test] +fn max_batch_size_constant() { + assert_eq!(MAX_BATCH_SIZE, 200); +} diff --git a/tests/serialization_roundtrip.rs b/tests/serialization_roundtrip.rs new file mode 100644 index 0000000..71850cf --- /dev/null +++ b/tests/serialization_roundtrip.rs @@ -0,0 +1,225 @@ +//! Round-trip persistence tests for the on-disk side caches. +//! +//! `ImmutableDataCache` (token decimals + pool metadata) and, under the +//! `protocols` feature, `V3TickSnapshotCache` are serialized with bincode and +//! reloaded across runs. These modules had no test coverage; the tests here pin +//! that a save/load cycle preserves the data, that a missing file is reported as +//! "no cache", and the current (silent-drop) behavior of the string-keyed V3 tick +//! snapshot — see `docs/KNOWN_ISSUES.md`. +//! +//! Files are written under the system temp directory and cleaned up, following +//! the dependency-free pattern used by the `binary_state` unit tests. + +use std::path::PathBuf; + +use alloy_primitives::{Address, B256, U256}; +use evm_fork_cache::cache::{ + BalancerPoolMetadata, ImmutableDataCache, V2PoolMetadata, V3PoolMetadata, +}; + +/// A unique temp directory for one test, removed on drop so a failing assertion +/// still cleans up. +struct TempDir(PathBuf); + +impl TempDir { + fn new(tag: &str) -> Self { + let dir = std::env::temp_dir().join(format!("evm_fork_cache_roundtrip_{tag}")); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("create temp dir"); + TempDir(dir) + } + + fn path(&self, file: &str) -> PathBuf { + self.0.join(file) + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +#[test] +fn immutable_data_cache_round_trips() { + let dir = TempDir::new("immutable"); + let path = dir.path("immutable_data.bin"); + + let token_a = Address::repeat_byte(0xA1); + let token_b = Address::repeat_byte(0xB2); + let v2_pool = Address::repeat_byte(0x22); + let v3_pool = Address::repeat_byte(0x33); + let balancer_id = B256::repeat_byte(0x44); + + let mut cache = ImmutableDataCache::default(); + assert!(cache.is_empty()); + + cache.set_token_decimals(token_a, 6); + cache.set_token_decimals(token_b, 18); + cache.set_v2_pool( + v2_pool, + V2PoolMetadata { + token0: token_a, + token1: token_b, + last_block_timestamp: 1_700_000_000, + }, + ); + cache.set_v3_pool( + v3_pool, + V3PoolMetadata { + token0: token_a, + token1: token_b, + fee: 3000, + tick_spacing: 60, + }, + ); + cache.set_balancer_pool( + balancer_id, + BalancerPoolMetadata { + tokens: vec![token_a, token_b], + weights: vec![U256::from(80u64), U256::from(20u64)], + swap_fee: U256::from(1_000u64), + last_change_block: U256::from(18_000_000u64), + }, + ); + + assert!(!cache.is_empty()); + let len_before = cache.len(); + + cache.save(&path).expect("save immutable cache"); + let loaded = ImmutableDataCache::load(&path).expect("load immutable cache"); + + // Counts and scalar values survive the round trip. + assert_eq!(loaded.len(), len_before); + assert_eq!(loaded.get_token_decimals(token_a), Some(6)); + assert_eq!(loaded.get_token_decimals(token_b), Some(18)); + assert_eq!(loaded.get_token_decimals(Address::ZERO), None); + + // Metadata structs do not derive PartialEq, so compare field-by-field. + let v2 = loaded.get_v2_pool(v2_pool).expect("v2 pool present"); + assert_eq!(v2.token0, token_a); + assert_eq!(v2.token1, token_b); + assert_eq!(v2.last_block_timestamp, 1_700_000_000); + + let v3 = loaded.get_v3_pool(v3_pool).expect("v3 pool present"); + assert_eq!(v3.token0, token_a); + assert_eq!(v3.token1, token_b); + assert_eq!(v3.fee, 3000); + assert_eq!(v3.tick_spacing, 60); + + // The Balancer pool is keyed by the id's Debug formatting; a lookup with the + // same B256 after reload must still resolve. + let bal = loaded + .get_balancer_pool(balancer_id) + .expect("balancer pool present after reload (Debug-key round trip)"); + assert_eq!(bal.tokens, vec![token_a, token_b]); + assert_eq!(bal.weights, vec![U256::from(80u64), U256::from(20u64)]); + assert_eq!(bal.swap_fee, U256::from(1_000u64)); + assert_eq!(bal.last_change_block, U256::from(18_000_000u64)); +} + +#[test] +fn immutable_data_cache_load_missing_file_is_none() { + let dir = TempDir::new("immutable_missing"); + let missing = dir.path("does_not_exist.bin"); + assert!(ImmutableDataCache::load(&missing).is_none()); +} + +#[test] +fn immutable_data_cache_load_corrupt_file_is_none() { + let dir = TempDir::new("immutable_corrupt"); + let path = dir.path("corrupt.bin"); + std::fs::write(&path, b"not valid bincode at all").expect("write corrupt file"); + // A decode failure is swallowed and reported as "no cache" (see KNOWN_ISSUES). + assert!(ImmutableDataCache::load(&path).is_none()); +} + +#[cfg(feature = "protocols")] +mod tick_snapshots { + use super::*; + use std::collections::HashMap; + + use evm_fork_cache::cache::{TickInfo, V3PoolTickSnapshot, V3TickSnapshotCache}; + + #[test] + fn v3_tick_snapshot_round_trips_including_negative_keys() { + let dir = TempDir::new("v3_ticks"); + let path = dir.path("v3_tick_snapshots.bin"); + let pool = Address::repeat_byte(0x77); + + // Word positions and tick indices are signed; include negatives, which + // are exactly where the string-key encoding could go wrong. + let mut bitmap: HashMap = HashMap::new(); + bitmap.insert(-3, U256::from(0b1010u64)); + bitmap.insert(0, U256::from(1u64)); + bitmap.insert(5, U256::from(u128::MAX)); + + let mut ticks: HashMap = HashMap::new(); + ticks.insert( + -887_272, + TickInfo { + liquidity_gross: 1_000, + liquidity_net: -500, + initialized: true, + }, + ); + ticks.insert( + 60, + TickInfo { + liquidity_gross: 42, + liquidity_net: 7, + initialized: false, + }, + ); + + let snapshot = V3PoolTickSnapshot::from_pool_data(&bitmap, &ticks, 12_345u128, -120); + + let mut cache = V3TickSnapshotCache::default(); + assert!(cache.is_empty()); + cache.set(pool, snapshot); + assert_eq!(cache.len(), 1); + + cache.save(&path).expect("save tick cache"); + let loaded = V3TickSnapshotCache::load(&path).expect("load tick cache"); + + let snap = loaded.get(pool).expect("snapshot present"); + assert_eq!(snap.last_liquidity, 12_345u128); + assert_eq!(snap.last_tick, -120); + // TickInfo derives PartialEq/Eq, so the recovered maps compare directly. + assert_eq!(snap.to_tick_bitmap(), bitmap, "bitmap survives round trip"); + assert_eq!(snap.to_ticks(), ticks, "ticks survive round trip"); + } + + #[test] + fn v3_tick_snapshot_silently_drops_unparseable_keys() { + // Pin the documented behavior (KNOWN_ISSUES): a string key that does not + // parse as the expected integer type is dropped without error. + let mut snapshot = V3PoolTickSnapshot::from_pool_data( + &HashMap::from([(1i16, U256::from(9u64))]), + &HashMap::new(), + 0, + 0, + ); + snapshot + .tick_bitmap + .insert("not-a-number".to_string(), U256::from(123u64)); + + let recovered = snapshot.to_tick_bitmap(); + assert_eq!(recovered.len(), 1, "the unparseable key is dropped"); + assert_eq!(recovered.get(&1i16), Some(&U256::from(9u64))); + } + + #[test] + fn v3_tick_snapshot_cache_remove() { + let pool = Address::repeat_byte(0x01); + let mut cache = V3TickSnapshotCache::default(); + cache.set( + pool, + V3PoolTickSnapshot::from_pool_data(&HashMap::new(), &HashMap::new(), 0, 0), + ); + assert_eq!(cache.len(), 1); + cache.remove(pool); + assert!(cache.is_empty()); + assert!(cache.get(pool).is_none()); + } +} diff --git a/tests/snapshot_overlay.rs b/tests/snapshot_overlay.rs new file mode 100644 index 0000000..3b18abc --- /dev/null +++ b/tests/snapshot_overlay.rs @@ -0,0 +1,171 @@ +//! Offline integration tests for the snapshot/overlay isolation guarantees that +//! underpin the crate's parallel fan-out model. +//! +//! These pin the invariants a search loop relies on: +//! - [`EvmCache::create_snapshot`] yields an immutable, point-in-time view that +//! later cache mutations cannot perturb. +//! - Overlays derived from one snapshot are isolated from each other and from the +//! live cache. +//! +//! All state is injected over a mocked provider, so no test touches the network. + +mod common; + +use std::sync::Arc; + +use alloy_primitives::{Address, U256, keccak256}; +use alloy_sol_types::{SolCall, SolValue}; +use anyhow::{Result, anyhow}; +use revm::context::result::ExecutionResult; +use revm::database_interface::Database; + +use common::{ + MOCK_ERC20_BALANCE_SLOT, MockERC20, install_default_account, install_mock_erc20, setup_cache, + transfer, +}; +use evm_fork_cache::cache::{EvmOverlay, EvmSnapshot}; + +/// The hashed storage slot of `balanceOf[owner]` for a `MockERC20` (balances at +/// the declared mapping slot 3): `keccak256(abi.encode(owner, 3))`. +fn balance_slot_for(owner: Address) -> U256 { + let key = keccak256((owner, U256::from(MOCK_ERC20_BALANCE_SLOT)).abi_encode()); + U256::from_be_bytes(key.0) +} + +/// Read `balanceOf(owner)` from a `MockERC20` through an overlay (non-committing). +fn overlay_balance_of(overlay: &mut EvmOverlay, token: Address, owner: Address) -> Result { + let call = MockERC20::balanceOfCall { account: owner }; + let result = overlay.call_raw(owner, token, call.abi_encode().into())?; + match result { + ExecutionResult::Success { output, .. } => Ok( + MockERC20::balanceOfCall::abi_decode_returns(&output.into_data())?, + ), + other => Err(anyhow!("overlay balanceOf failed: {other:?}")), + } +} + +/// A snapshot captures state at a point in time; committing a transfer on the +/// live cache afterward must not change what an overlay built from that snapshot +/// observes. +#[tokio::test(flavor = "multi_thread")] +async fn snapshot_is_immutable_after_later_cache_mutation() -> Result<()> { + let mut cache = setup_cache().await?; + let token = Address::repeat_byte(0x11); + let owner = Address::repeat_byte(0x22); + let recipient = Address::repeat_byte(0x33); + + install_default_account(&mut cache, Address::ZERO); + install_default_account(&mut cache, owner); + install_default_account(&mut cache, recipient); + install_mock_erc20(&mut cache, token); + + let balance_slot = U256::from(MOCK_ERC20_BALANCE_SLOT); + let initial = U256::from(1_000u64); + cache.insert_mapping_storage_slot(token, balance_slot, owner, initial)?; + cache.insert_mapping_storage_slot(token, balance_slot, recipient, U256::ZERO)?; + + // Freeze the state, then mutate the live cache with a committed transfer. + let snapshot = cache.create_snapshot(); + transfer(&mut cache, token, owner, recipient, U256::from(250u64))?; + + // The live cache reflects the transfer... + assert_eq!( + common::balance_of(&mut cache, token, owner)?, + initial - U256::from(250u64), + "live cache should reflect the committed transfer" + ); + + // ...but the snapshot (and any overlay built from it) is frozen at `initial`. + assert_eq!( + snapshot.storage_value(token, balance_slot_for(owner)), + Some(initial), + "snapshot storage_value is unaffected by the later mutation" + ); + let mut overlay = EvmOverlay::new(Arc::clone(&snapshot), None); + assert_eq!( + overlay_balance_of(&mut overlay, token, owner)?, + initial, + "overlay from the snapshot sees the pre-transfer balance" + ); + + Ok(()) +} + +/// Two overlays built from the same snapshot are isolated: a dirty-layer write in +/// one is invisible to the other and to the live cache. +#[tokio::test(flavor = "multi_thread")] +async fn overlays_from_one_snapshot_are_isolated() -> Result<()> { + let mut cache = setup_cache().await?; + let contract = Address::repeat_byte(0x99); + install_mock_erc20(&mut cache, contract); + + let slot = U256::from(7); + let original = U256::from(1u64); + cache.inject_storage_batch(&[(contract, slot, original)]); + + let snapshot = cache.create_snapshot(); + let mut overlay_a = EvmOverlay::new(Arc::clone(&snapshot), None); + let mut overlay_b = EvmOverlay::new(Arc::clone(&snapshot), None); + + // Write through overlay A only. + overlay_a.override_slot(contract, slot, U256::from(999u64)); + + assert_eq!( + overlay_a.storage(contract, slot)?, + U256::from(999u64), + "overlay A sees its own dirty-layer write" + ); + assert_eq!( + overlay_b.storage(contract, slot)?, + original, + "overlay B is isolated from overlay A's write" + ); + assert_eq!( + cache.cached_storage_value(contract, slot), + Some(original), + "the live cache is unaffected by an overlay write" + ); + assert_eq!( + snapshot.storage_value(contract, slot), + Some(original), + "the shared snapshot is unaffected by an overlay write" + ); + + Ok(()) +} + +/// A fresh overlay (no dirty-layer writes) reads exactly the snapshot's state. +#[tokio::test(flavor = "multi_thread")] +async fn overlay_reads_reflect_snapshot_state() -> Result<()> { + let mut cache = setup_cache().await?; + let token = Address::repeat_byte(0x44); + let owner = Address::repeat_byte(0x55); + + install_default_account(&mut cache, Address::ZERO); + install_default_account(&mut cache, owner); + install_mock_erc20(&mut cache, token); + cache.insert_mapping_storage_slot( + token, + U256::from(MOCK_ERC20_BALANCE_SLOT), + owner, + U256::from(42_000u64), + )?; + + let snapshot: Arc = cache.create_snapshot(); + let mut overlay = EvmOverlay::new(snapshot, None); + + assert_eq!( + overlay_balance_of(&mut overlay, token, owner)?, + U256::from(42_000u64) + ); + + // A non-committing overlay call leaves the overlay's base state intact, so a + // repeat read returns the same value. + assert_eq!( + overlay_balance_of(&mut overlay, token, owner)?, + U256::from(42_000u64), + "overlay calls are non-committing" + ); + + Ok(()) +} diff --git a/tests/storage_keys.rs b/tests/storage_keys.rs index a6ac118..553cace 100644 --- a/tests/storage_keys.rs +++ b/tests/storage_keys.rs @@ -1,5 +1,10 @@ //! Tests for the Uniswap V3-style storage-key derivation helpers, exercised //! through their public re-export path so the coverage travels with the crate. +//! +//! Gated on the `protocols` feature: the helpers under test are only compiled +//! (and re-exported) when that feature is on, so without it this whole file is +//! cfg'd out rather than failing to build under `--no-default-features`. +#![cfg(feature = "protocols")] use alloy_primitives::U256; use evm_fork_cache::cache::{v3_tick_bitmap_storage_key, v3_tick_info_storage_keys}; diff --git a/tests/transfer_inspector.rs b/tests/transfer_inspector.rs new file mode 100644 index 0000000..9e65c21 --- /dev/null +++ b/tests/transfer_inspector.rs @@ -0,0 +1,194 @@ +//! End-to-end integration tests for transfer-tracking simulation. +//! +//! The inline unit tests in `src/inspector.rs` populate the inspector by hand; +//! these drive it through a real EVM execution — `MockERC20.transfer` emits a +//! `Transfer` event that the [`TransferInspector`](evm_fork_cache::inspector::TransferInspector) +//! captures during [`EvmCache::simulate_with_transfer_tracking`] — and assert the +//! reconstructed balance deltas, log capture, token filtering, non-committing +//! semantics, and the revert path. All state is injected over a mocked provider. + +mod common; + +use alloy_primitives::{Address, I256, U256}; +use alloy_sol_types::SolCall; +use anyhow::Result; + +use common::{ + MOCK_ERC20_BALANCE_SLOT, MockERC20, install_default_account, install_mock_erc20, setup_cache, +}; +use evm_fork_cache::errors::RevertReason; + +/// Build the calldata for `transfer(to, amount)`. +fn transfer_calldata(to: Address, amount: U256) -> alloy_primitives::Bytes { + MockERC20::transferCall { to, amount }.abi_encode().into() +} + +/// A transfer the inspector observes yields a signed delta for the sender, the +/// emitted `Transfer` log is captured, and the populated access list reflects the +/// touched token. The non-committing sim leaves the on-chain balance unchanged. +#[tokio::test(flavor = "multi_thread")] +async fn transfer_tracking_reports_sender_delta_and_logs() -> Result<()> { + let mut cache = setup_cache().await?; + let token = Address::repeat_byte(0x11); + let owner = Address::repeat_byte(0x22); + let recipient = Address::repeat_byte(0x33); + + install_default_account(&mut cache, Address::ZERO); + install_default_account(&mut cache, owner); + install_default_account(&mut cache, recipient); + install_mock_erc20(&mut cache, token); + + let balance_slot = U256::from(MOCK_ERC20_BALANCE_SLOT); + cache.insert_mapping_storage_slot(token, balance_slot, owner, U256::from(1_000u64))?; + cache.insert_mapping_storage_slot(token, balance_slot, recipient, U256::ZERO)?; + + let result = cache.simulate_with_transfer_tracking( + owner, + token, + transfer_calldata(recipient, U256::from(250u64)), + owner, + Some([token]), + false, // non-committing + )?; + + // Owner sent 250 of `token`. + assert_eq!( + result.token_deltas.get(&token), + Some(&I256::try_from(-250i64).unwrap()), + "sender's delta is -amount" + ); + // The Transfer log was captured. + assert_eq!(result.logs.len(), 1, "exactly one Transfer log emitted"); + // The inspector path also captures the EIP-2930 access list. + assert!( + result + .access_list + .0 + .iter() + .any(|item| item.address == token), + "access list includes the token account" + ); + + // Non-committing: the on-chain balance is untouched. + assert_eq!( + common::balance_of(&mut cache, token, owner)?, + U256::from(1_000u64), + "a non-committing sim must not change cache state" + ); + + Ok(()) +} + +/// The recipient's perspective sees the mirror-image positive delta. +#[tokio::test(flavor = "multi_thread")] +async fn transfer_tracking_reports_recipient_delta() -> Result<()> { + let mut cache = setup_cache().await?; + let token = Address::repeat_byte(0x44); + let owner = Address::repeat_byte(0x55); + let recipient = Address::repeat_byte(0x66); + + install_default_account(&mut cache, Address::ZERO); + install_default_account(&mut cache, owner); + install_default_account(&mut cache, recipient); + install_mock_erc20(&mut cache, token); + cache.insert_mapping_storage_slot( + token, + U256::from(MOCK_ERC20_BALANCE_SLOT), + owner, + U256::from(500u64), + )?; + + // `owner` argument selects whose deltas to compute — here, the recipient. + let result = cache.simulate_with_transfer_tracking( + owner, + token, + transfer_calldata(recipient, U256::from(120u64)), + recipient, + None::>, + false, + )?; + + assert_eq!( + result.token_deltas.get(&token), + Some(&I256::try_from(120i64).unwrap()), + "recipient's delta is +amount" + ); + + Ok(()) +} + +/// The `tokens` filter restricts which tokens appear in the deltas: a transfer in +/// a token absent from the filter set is dropped from the result. +#[tokio::test(flavor = "multi_thread")] +async fn transfer_tracking_token_filter_excludes_other_tokens() -> Result<()> { + let mut cache = setup_cache().await?; + let token = Address::repeat_byte(0x77); + let other_token = Address::repeat_byte(0x78); + let owner = Address::repeat_byte(0x88); + let recipient = Address::repeat_byte(0x89); + + install_default_account(&mut cache, Address::ZERO); + install_default_account(&mut cache, owner); + install_default_account(&mut cache, recipient); + install_mock_erc20(&mut cache, token); + cache.insert_mapping_storage_slot( + token, + U256::from(MOCK_ERC20_BALANCE_SLOT), + owner, + U256::from(1_000u64), + )?; + + // Filter to a different token than the one transferred. + let result = cache.simulate_with_transfer_tracking( + owner, + token, + transfer_calldata(recipient, U256::from(250u64)), + owner, + Some([other_token]), + false, + )?; + + assert!( + result.token_deltas.is_empty(), + "the transferred token is filtered out, leaving no deltas" + ); + + Ok(()) +} + +/// An insufficient-balance transfer reverts; the typed error surfaces the decoded +/// `Error("balance")` reason rather than a generic failure. +#[tokio::test(flavor = "multi_thread")] +async fn transfer_tracking_surfaces_revert_reason() -> Result<()> { + let mut cache = setup_cache().await?; + let token = Address::repeat_byte(0xAA); + let owner = Address::repeat_byte(0xBB); + let recipient = Address::repeat_byte(0xCC); + + install_default_account(&mut cache, Address::ZERO); + install_default_account(&mut cache, owner); + install_default_account(&mut cache, recipient); + install_mock_erc20(&mut cache, token); + // owner has zero balance, so transferring reverts in `_transfer`'s require. + + let err = cache + .simulate_with_transfer_tracking( + owner, + token, + transfer_calldata(recipient, U256::from(100u64)), + owner, + None::>, + false, + ) + .expect_err("transfer with no balance must revert"); + + assert!(err.is_revert(), "expected a revert, got {err:?}"); + let revert = err.as_revert().expect("revert payload"); + assert_eq!( + revert.reason(), + &RevertReason::Error("balance".to_string()), + "MockERC20._transfer reverts with require(.., \"balance\")" + ); + + Ok(()) +} From 81efa6b776402b9a1f29c122cf08ff3d1d55aa9e Mon Sep 17 00:00:00 2001 From: Kai Aldag Date: Tue, 16 Jun 2026 02:37:09 +0100 Subject: [PATCH 11/26] Phase 3: state-update primitives (Pillar B.1) + relative RMW + audit remediation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the Pillar B.1 state-update layer: a generic `StateUpdate` vocabulary the (Phase 4) event decoder will emit, applied through one unified write-through path, with a structured `StateDiff` output. Existing writers (`inject_*`, `purge_*`, `override_account_code*`) and the freshness pending-drain are refolded onto that path. Vocabulary & apply: - `StateUpdate::{Slot, SlotDelta, Account, Purge, BalanceDelta}` + constructors (`slot`, `slot_delta`, `balance`, `balance_delta`, `nonce`, `code`, `account`, `purge`); `AccountPatch` (partial balance/nonce/code); `PurgeScope`. - `EvmCache::apply_update` / `apply_updates` -> `StateDiff { slots, accounts, purged, skipped, skipped_balances }` with `is_empty`/`len` (changes-only), `has_skipped`/`skipped_len`/`is_fully_applied`, and `merge`. Relative read-modify-write (cold-aware): - `SlotDelta::{Add,Sub}` (saturating) for storage; `BalanceDelta` for native balance; closure escape hatches `modify_slot` / `modify_account_balance`. - A delta against a value the cache does not hold (cold) is NOT applied — it is surfaced in `skipped` / `skipped_balances` so the caller can seed the truth, never corrupting an unknown base. This powers event-driven balance tracking (index a Transfer -> `[Sub on from, Add on to]`). Audit remediation (5-lens adversarial audit; see docs/phase-3-spec.md §16): - FIX (HIGH, silent corruption): `cached_storage_value` is now `account_state`- aware — a slot absent from a `StorageCleared`/`NotExisting` overlay account reads as ZERO (mirroring the EVM SLOAD / `CacheDB::storage_ref`) instead of returning a shadowed backend value. Pre-fix, a relative update computed against a base the EVM never sees. Pinned by an SLOAD-validated reproducer. - FIX (no-op Account patch no longer materializes a backend account). - `serde` on the whole vocabulary + `freshness::SlotChange`; `#[non_exhaustive]` on `StateDiff` + `AccountPatch` (leaf record types kept exhaustive so callers can still build them for equality assertions). - Perf: batched single-lock fast-path for runs of `Slot`/`SlotDelta` writes in `apply_updates` (one backend storage write-guard per run; dropped before `Account`/`BalanceDelta`/`Purge`), plus elimination of the `SlotDelta` double-read. Validated byte-for-byte against the sequential fold. Tests/docs/benches: tests/state_update.rs (49) covering every variant, the cold-skip guarantee, write-through layering, the batched==sequential equivalence net, serde round-trip, and the protocols Decision-2 write-through pins; the freshness/snapshot seeds were corrected to be EVM-visible (overlay-resident) since a backend-only seed on a StorageCleared account is invisible post-fix. benches/state_update.rs, examples/state_update_apply.rs, CHANGELOG, ROADMAP, KNOWN_ISSUES, README updated. Full suite green (250 tests + 31 doctests), clippy default + --no-default-features, fmt, RUSTDOCFLAGS=-D warnings doc, and cargo bench --no-run all clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 82 ++ Cargo.toml | 4 + README.md | 2 + benches/state_update.rs | 318 +++++++ docs/KNOWN_ISSUES.md | 32 +- docs/ROADMAP.md | 54 +- docs/phase-3-spec.md | 860 ++++++++++++++++++ examples/state_update_apply.rs | 183 ++++ src/cache/mod.rs | 737 ++++++++++++++- src/freshness.rs | 27 +- src/lib.rs | 9 + src/state_update.rs | 696 +++++++++++++++ tests/freshness.rs | 45 +- tests/snapshot_overlay.rs | 10 +- tests/state_update.rs | 1528 ++++++++++++++++++++++++++++++++ 15 files changed, 4532 insertions(+), 55 deletions(-) create mode 100644 benches/state_update.rs create mode 100644 docs/phase-3-spec.md create mode 100644 examples/state_update_apply.rs create mode 100644 src/state_update.rs create mode 100644 tests/state_update.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index c3229e8..8fe44fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,60 @@ pre-release development phases (see [`docs/ROADMAP.md`](docs/ROADMAP.md)). - **Freshness primitives on `EvmCache`** — `verify_slots`, `purge_account`, `set_storage_batch_fetcher`; `EvmOverlay::call_raw_with_access_list` and `override_slot` for read-set capture and corrected re-runs. +- **State-update vocabulary & apply primitive** (`state_update` module, Phase 3, + Pillar B.1) — a generic `StateUpdate` enum (`Slot` / partial-`AccountPatch` + `Account` / `Purge` by `PurgeScope`) plus `EvmCache::apply_update` / + `apply_updates`, the single dual-layer write-through primitive (backend always, + overlay-if-present, no new overlay account materialized), returning a structured + `StateDiff` (`SlotChange`s, `AccountChange`s, `PurgeRecord`s) that records only + actual changes. The existing `inject_storage_batch_fresh` / `purge_account` / + `purge_pool_storage` / `purge_pool_slots` writers and the freshness + correction-drain are refolded onto it (signatures unchanged); generic, builds + with `--no-default-features`. +- **Relative / read-modify-write state updates** (`state_update`, Phase 3 §15) — + a saturating `SlotDelta` (`Add`/`Sub`, clamping at `U256::MAX`/`U256::ZERO`), a + `StateUpdate::SlotDelta { address, slot, delta }` variant (with the + `StateUpdate::slot_delta` constructor) so deltas flow through `apply_updates`, + and `EvmCache::modify_slot(address, slot, |Option| -> Option)` as + the general closure escape hatch. Relative application is **cold-aware**: a + delta against a slot absent from both layers is not applied (it would corrupt an + unknown value) but surfaced in the new `StateDiff.skipped: Vec` + field for the caller to fetch+seed and retry. `skipped` is informational + metadata and does not affect `StateDiff::is_empty` / `len` (changes-only). + Adding the `StateDiff.skipped` field is a struct change permitted under the + pre-1.0 break policy. Generic core (builds `--no-default-features`). +- **Post-audit state-update remediation** (`state_update`, Phase 3 §16): + - **`serde`** — `Serialize`/`Deserialize` derived (unconditionally) on the whole + vocabulary (`SlotDelta`, `StateUpdate`, `AccountPatch`, `PurgeScope`) and the + diff (`StateDiff`, `AccountChange`, `PurgeRecord`, `SkippedDelta`, + `SkippedBalanceDelta`) plus `freshness::SlotChange`, so updates can be shipped + over the wire and diffs persisted. + - **`#[non_exhaustive]`** on `StateDiff` and `AccountPatch` (both + `Default`/builder-constructed), so future field additions are non-breaking. The + leaf record types (`SlotChange`/`AccountChange`/`PurgeRecord`/`SkippedDelta`/ + `SkippedBalanceDelta`) are deliberately left exhaustive — they are routinely + built as struct literals in equality assertions. + - **Relative native-balance updates** — a `StateUpdate::BalanceDelta { address, + delta: SlotDelta }` variant (with `StateUpdate::balance_delta`), the + `EvmCache::modify_account_balance(addr, |Option| -> Option)` + closure escape hatch, a new `StateDiff.skipped_balances: + Vec` field, and `SkippedBalanceDelta`. Cold-aware: a delta + on an account absent from both layers is skipped and surfaced (never + materialized). Adding `skipped_balances` is a struct change permitted under the + pre-1.0 break policy. + - **Discoverable skip accessors** — `StateDiff::has_skipped()` / `skipped_len()` + / `is_fully_applied()`, counting **both** `skipped` and `skipped_balances`, so a + silently-dropped cold relative update is easy to detect (the changes-only + `is_empty()`/`len()` do not reflect skips). + - **Constructor symmetry** — `StateUpdate::nonce(addr, u64)`, + `StateUpdate::code(addr, Bytes)`, `StateUpdate::account(addr, AccountPatch)`. +- **Batched single-lock fast-path for `apply_updates`** (Phase 3 §16.9): a run of + consecutive `Slot`/`SlotDelta` writes now holds the backend storage write-guard + once for the run (the guard is dropped before any `Account`/`BalanceDelta`/ + `Purge` update to avoid deadlocking the non-reentrant `RwLock`, then re-acquired), + and the `SlotDelta` double-read of the old value is eliminated. The result is + byte-identical to folding `apply_update` over the batch (pinned by the + batched==sequential equivalence test). Generic core. - **Configurable transaction & block environment** — `TxConfig` (value, gas limit, gas price, nonce, access list) threaded through `call_raw_with`; block context setters (`set_coinbase`, `set_prevrandao`, `set_block_gas_limit`). @@ -58,6 +112,34 @@ pre-release development phases (see [`docs/ROADMAP.md`](docs/ROADMAP.md)). - Simulation entry points that distinguish failure modes return `SimulationResult` (`Result`), separating decoded reverts, EVM halts, and host errors. `SimulationErrorKind` remains as a deprecated alias. +- **`inject_v2_pool_metadata` / `inject_v3_tick_bitmap*` / `inject_v3_ticks*` + (`protocols`) now write through both cache layers** (Phase 3, Decision 2). + Previously these wrote only the CacheDB overlay (layer 1); they are now folded + onto the write-through `StateUpdate::Slot` primitive, so the injected slots also + land in the BlockchainDb backend (layer 2). Signatures and return values are + unchanged and the visible `token0()`/`tickBitmap()`/`ticks()` reads are the + same; only the slot *placement* across layers changed. See + [`docs/KNOWN_ISSUES.md`](docs/KNOWN_ISSUES.md). (The cold-backfill + `inject_storage_batch` keeps its layer-2-only intent and is unchanged.) + +### Fixed + +- **`cached_storage_value` silent-corruption bug** (Phase 3 §16.0, audit HIGH + + MED). For a storage slot absent from an overlay account whose revm + `account_state` is `StorageCleared` or `NotExisting`, the accessor now returns + `Some(U256::ZERO)` — mirroring what the live EVM `SLOAD`s + (`CacheDB::storage_ref`) — instead of falling through to the BlockchainDb + backend and returning a *shadowed* backend value the EVM never sees. The old + behavior let a `SlotDelta` / `modify_slot` compute a relative update against a + base the EVM never reads (silent state corruption) and mis-recorded + `apply_slot`'s `SlotChange.old` / change predicate. This also closes the + same-root mismatch shared by `verify_slots` / `inject_storage_batch_fresh`. +- **No-op `Account` patch no longer materializes a backend account** (Phase 3 + §16.1, audit LOW). `apply_account_patch` now computes the field change first and + **skips both layer writes** (returning an empty diff) when no field actually + changes, instead of unconditionally inserting `AccountInfo::default()` into the + shared backend for an all-`None` (or value-unchanged) patch on an absent address. + A real field change still materializes the backend account (unchanged intent). ### Notes diff --git a/Cargo.toml b/Cargo.toml index 6ccbb62..cdde9d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -86,6 +86,10 @@ harness = false name = "freshness" harness = false +[[bench]] +name = "state_update" +harness = false + # RPC-gated real-contract benchmarks. Skipped (not failed) when RPC_URL is unset, # so `cargo bench` stays offline by default. [[bench]] diff --git a/README.md b/README.md index 3921531..0d1b959 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,7 @@ and inject all state directly: | `prefetch_registry` | Advanced | Record and persist storage touch sets for cross-cycle prefetch. | | `freshness_optimistic` | Advanced | Optimistic verify-and-rerun loop: a `Corrected` validation via a stub fetcher. | | `freshness_multi_sim` | Advanced | Many sims with selective re-run, plus classification and `ValidThrough` aging. | +| `state_update_apply` | Advanced | Apply a mixed `StateUpdate` batch (`Slot`/`Account`/`Purge`) and inspect the returned `StateDiff`. | **RPC examples** fork real mainnet state. Set `RPC_URL` to an Ethereum RPC endpoint (they print instructions and exit if it is unset): @@ -224,6 +225,7 @@ cache sizes: | --- | --- | | `simulation` | `create_snapshot` across cache sizes (100 → 10k accounts), overlay fan-out, `call_raw` throughput, sequential bundle execution, batched storage injection. | | `freshness` | The optimistic loop end-to-end (CPU and latency-hiding), `verify_slots` at scale (1 → 1000 slots), and multi-sim fan-out. | +| `state_update` | `apply_updates` throughput across batch sizes (1 → 1000 `Slot`s) and per-variant apply cost (`Slot` vs `Account` vs `Purge`). | | `access_list` | Touch-set merge and EIP-2930 list construction. | | `revert_decoding` | Built-in and custom revert decoding, including decoder dispatch with many registered errors. | | `storage_keys` | Mapping/array storage-key derivation. | diff --git a/benches/state_update.rs b/benches/state_update.rs new file mode 100644 index 0000000..907954c --- /dev/null +++ b/benches/state_update.rs @@ -0,0 +1,318 @@ +//! Phase 3 benchmarks: the targeted state-update apply primitive. +//! +//! Measures [`EvmCache::apply_updates`] throughput across batch sizes +//! (1 → 1000 `Slot` writes) and the per-variant cost of a single apply (`Slot` +//! vs `Account` patch vs `Purge`). The cache is built once per group; each +//! iteration re-uses it (the writes are idempotent / additive in-memory). +//! +//! Fully offline (mocked provider, state injected directly), so reproducible. +//! A current-thread runtime is used only to drive the async cache constructor; +//! `apply_updates` itself is synchronous and never touches the network. + +use std::hint::black_box; +use std::sync::Arc; + +use alloy_primitives::{Address, Bytes, U256, hex}; +use alloy_provider::RootProvider; +use alloy_provider::network::AnyNetwork; +use alloy_rpc_client::RpcClient; +use alloy_transport::mock::Asserter; +use criterion::{BatchSize, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use evm_fork_cache::cache::EvmCache; +use evm_fork_cache::{AccountPatch, PurgeScope, SlotDelta, StateUpdate}; +use revm::state::{AccountInfo, Bytecode}; +use tokio::runtime::{Builder, Runtime}; + +const MOCK_ERC20_RUNTIME_HEX: &str = include_str!("../fixtures/mock_erc20_runtime.hex"); +const POOL: Address = Address::repeat_byte(0xAA); + +fn current_thread_rt() -> Runtime { + Builder::new_current_thread().enable_all().build().unwrap() +} + +/// A cache with `POOL` installed as a MockERC20 (overlay account present, so slot +/// writes exercise the overlay write-through branch too). +fn pool_cache(rt: &Runtime) -> EvmCache { + let provider = RootProvider::::new(RpcClient::mocked(Asserter::new())); + let mut cache = rt.block_on(EvmCache::new(Arc::new(provider), None)); + let runtime = Bytecode::new_raw(Bytes::from( + hex::decode(MOCK_ERC20_RUNTIME_HEX.trim()).unwrap(), + )); + let code_hash = runtime.hash_slow(); + cache.db_mut().insert_account_info( + POOL, + AccountInfo { + balance: U256::ZERO, + nonce: 0, + code: Some(runtime), + code_hash, + account_id: None, + }, + ); + // Mark storage local so unseeded slots read as zero (no RPC fallthrough). + cache + .db_mut() + .replace_account_storage(POOL, Default::default()) + .unwrap(); + cache +} + +/// `apply_updates` throughput as the `Slot` batch grows (1 → 1000). +fn bench_apply_slots_batch(c: &mut Criterion) { + let rt = current_thread_rt(); + let mut cache = pool_cache(&rt); + + let mut group = c.benchmark_group("apply_slots_batch"); + for &n in &[1usize, 10, 100, 1_000] { + // A fresh value each iteration is unnecessary; alternate two values so + // every apply records a real change (the worst case: full diff). + let updates_a: Vec = (0..n) + .map(|i| StateUpdate::slot(POOL, U256::from(i as u64), U256::from(1u64))) + .collect(); + let updates_b: Vec = (0..n) + .map(|i| StateUpdate::slot(POOL, U256::from(i as u64), U256::from(2u64))) + .collect(); + + group.throughput(Throughput::Elements(n as u64)); + group.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, _| { + let mut toggle = false; + b.iter(|| { + let updates = if toggle { &updates_a } else { &updates_b }; + toggle = !toggle; + black_box(cache.apply_updates(black_box(updates))); + }) + }); + } + group.finish(); +} + +/// Per-variant cost of a single `apply_update`: `Slot` vs `Account` vs `Purge`. +fn bench_apply_per_variant(c: &mut Criterion) { + let rt = current_thread_rt(); + + let mut group = c.benchmark_group("apply_per_variant"); + + group.bench_function("slot", |b| { + let mut cache = pool_cache(&rt); + let mut toggle = false; + b.iter(|| { + let value = if toggle { U256::from(1) } else { U256::from(2) }; + toggle = !toggle; + black_box(cache.apply_update(black_box(&StateUpdate::slot( + POOL, + U256::from(0), + value, + )))); + }) + }); + + group.bench_function("account_balance", |b| { + let mut cache = pool_cache(&rt); + let mut toggle = false; + b.iter(|| { + let value = if toggle { U256::from(1) } else { U256::from(2) }; + toggle = !toggle; + black_box(cache.apply_update(black_box(&StateUpdate::Account { + address: POOL, + patch: AccountPatch::default().balance(value), + }))); + }) + }); + + // A relative SlotDelta on a hot slot (seeded once, additive each iter). + group.bench_function("slot_delta_hot", |b| { + let mut cache = pool_cache(&rt); + cache.inject_storage_batch(&[(POOL, U256::from(0), U256::from(1))]); + b.iter(|| { + // Add(0) keeps the value stable so the slot stays hot across iters. + black_box(cache.apply_update(black_box(&StateUpdate::slot_delta( + POOL, + U256::from(0), + SlotDelta::Add(U256::ZERO), + )))); + }) + }); + + // A relative SlotDelta on a cold slot (always skipped, never applied). + group.bench_function("slot_delta_cold", |b| { + let mut cache = pool_cache(&rt); + b.iter(|| { + // POOL is StorageCleared, so an unseeded slot reads ZERO (hot). Use a + // distinct address with no overlay account and no backend slot: cold. + black_box(cache.apply_update(black_box(&StateUpdate::slot_delta( + Address::repeat_byte(0xCD), + U256::from(0), + SlotDelta::Add(U256::from(1)), + )))); + }) + }); + + // The general closure read-modify-write escape hatch. + group.bench_function("modify_slot", |b| { + let mut cache = pool_cache(&rt); + cache.inject_storage_batch(&[(POOL, U256::from(0), U256::from(1))]); + b.iter(|| { + black_box(cache.modify_slot(POOL, U256::from(0), |cur| { + cur.map(|v| v.saturating_add(U256::ZERO)) + })); + }) + }); + + // An `Account` *code* patch: `Bytecode::new_raw` + `hash_slow` (a keccak over + // the code) — likely the most expensive single apply. Toggle two code blobs + // so each apply records a real change. + group.bench_function("account_code", |b| { + let mut cache = pool_cache(&rt); + let code_a = Bytes::from_static(&[0x60, 0x00, 0x60, 0x00, 0xf3]); + let code_b = Bytes::from_static(&[0x60, 0x01, 0x60, 0x01, 0xf3]); + let mut toggle = false; + b.iter(|| { + let code = if toggle { + code_a.clone() + } else { + code_b.clone() + }; + toggle = !toggle; + black_box(cache.apply_update(black_box(&StateUpdate::code(POOL, code)))); + }) + }); + + // Purge mutates the cache, so re-seed each iteration via iter_batched. + group.bench_function("purge_all_storage", |b| { + b.iter_batched( + || { + let cache = pool_cache(&rt); + cache.inject_storage_batch(&[ + (POOL, U256::from(0), U256::from(1)), + (POOL, U256::from(1), U256::from(2)), + (POOL, U256::from(2), U256::from(3)), + ]); + cache + }, + |mut cache| { + black_box( + cache + .apply_update(black_box(&StateUpdate::purge(POOL, PurgeScope::AllStorage))), + ); + }, + BatchSize::SmallInput, + ) + }); + + // `PurgeScope::Account` (full account + storage removal). + group.bench_function("purge_account", |b| { + b.iter_batched( + || { + let cache = pool_cache(&rt); + cache.inject_storage_batch(&[ + (POOL, U256::from(0), U256::from(1)), + (POOL, U256::from(1), U256::from(2)), + ]); + cache + }, + |mut cache| { + black_box( + cache.apply_update(black_box(&StateUpdate::purge(POOL, PurgeScope::Account))), + ); + }, + BatchSize::SmallInput, + ) + }); + + // `PurgeScope::Slots` (a few specific slots). + group.bench_function("purge_slots", |b| { + b.iter_batched( + || { + let cache = pool_cache(&rt); + cache.inject_storage_batch(&[ + (POOL, U256::from(0), U256::from(1)), + (POOL, U256::from(1), U256::from(2)), + (POOL, U256::from(2), U256::from(3)), + ]); + cache + }, + |mut cache| { + black_box(cache.apply_update(black_box(&StateUpdate::purge( + POOL, + PurgeScope::Slots(vec![U256::from(0), U256::from(2)]), + )))); + }, + BatchSize::SmallInput, + ) + }); + + group.finish(); +} + +/// A *heterogeneous* `apply_updates` batch (Slot + Account + Purge) — exercises +/// the single-lock fast-path drop/re-acquire discipline around the non-slot +/// updates. +fn bench_apply_heterogeneous(c: &mut Criterion) { + let rt = current_thread_rt(); + let mut group = c.benchmark_group("apply_updates_mixed"); + + group.bench_function("slot_account_purge", |b| { + b.iter_batched( + || { + let cache = pool_cache(&rt); + cache.inject_storage_batch(&[(POOL, U256::from(9), U256::from(1))]); + cache + }, + |mut cache| { + // The cache is re-seeded each iteration, so a fixed value still + // records real changes (slots start at ZERO, balance/purge act on + // the fresh seed). + let value = U256::from(2); + black_box(cache.apply_updates(black_box(&[ + StateUpdate::slot(POOL, U256::from(0), value), + StateUpdate::slot(POOL, U256::from(1), value), + StateUpdate::balance(POOL, value), + StateUpdate::purge(POOL, PurgeScope::Slots(vec![U256::from(9)])), + StateUpdate::slot(POOL, U256::from(2), value), + ]))); + }, + BatchSize::SmallInput, + ) + }); + + group.finish(); +} + +/// A *distinct-address* `apply_updates` batch — the only fair apples-to-apples +/// comparison against the raw `inject_storage_batch` baseline (each write targets +/// a different address, so no overlay account exists and the fast-path holds the +/// backend storage guard once for the whole run). +fn bench_apply_distinct_addresses(c: &mut Criterion) { + let rt = current_thread_rt(); + let mut group = c.benchmark_group("apply_distinct_addresses"); + + for &n in &[10usize, 100, 1_000] { + let updates_a: Vec = (0..n) + .map(|i| StateUpdate::slot(Address::repeat_byte(i as u8), U256::from(0), U256::from(1))) + .collect(); + let updates_b: Vec = (0..n) + .map(|i| StateUpdate::slot(Address::repeat_byte(i as u8), U256::from(0), U256::from(2))) + .collect(); + + group.throughput(Throughput::Elements(n as u64)); + group.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, _| { + let mut cache = pool_cache(&rt); + let mut toggle = false; + b.iter(|| { + let updates = if toggle { &updates_a } else { &updates_b }; + toggle = !toggle; + black_box(cache.apply_updates(black_box(updates))); + }) + }); + } + group.finish(); +} + +criterion_group!( + benches, + bench_apply_slots_batch, + bench_apply_per_variant, + bench_apply_heterogeneous, + bench_apply_distinct_addresses, +); +criterion_main!(benches); diff --git a/docs/KNOWN_ISSUES.md b/docs/KNOWN_ISSUES.md index 0d93a5d..ac00fb9 100644 --- a/docs/KNOWN_ISSUES.md +++ b/docs/KNOWN_ISSUES.md @@ -71,6 +71,21 @@ Confidence legend: **[V]** verified against the source during review; override is set, which panics if the system clock is before the Unix epoch. Setting an explicit timestamp avoids it; consider a saturating fallback. +18. **[V] Cold absolute `Account` patch masks the real on-chain account.** A + *partial* absolute [`StateUpdate::Account`] patch (e.g. balance-only) applied + to an address absent from **both** cache layers writes default values for the + un-patched fields (nonce `0`, empty code) through the shared BlockchainDb + backend as authoritative — pre-empting a later RPC fetch of the real account + (`apply_account_patch` materializes the backend account on any real change, by + design / spec §5.2). This is a live-fork footgun for callers reconstructing an + account from one event field. Mitigations: fetch+seed the account first, or use + the relative `StateUpdate::BalanceDelta` / `EvmCache::modify_account_balance` + (Phase 3 §16.5), which are cold-aware (a cold target is skipped and surfaced in + `StateDiff.skipped_balances`, never materialized). A no-op patch (no field + actually changes) does **not** materialize anything (Phase 3 §16.1 fix). The + rustdoc on `apply_update` / `StateUpdate::Account` / `AccountPatch` carries a + `# Warning` to this effect. + ## Code-quality nits 11. **[V] Dead branch in `i128_to_u256`** (`cache/storage_keys.rs`): both the @@ -124,7 +139,20 @@ Confidence legend: **[V]** verified against the source during review; planned (roadmap), blocked partly by `ImmutableDataCache` coupling generic token-decimals with V2/V3/Balancer pool metadata. - **Event-driven sync (roadmap Pillar B) is not implemented.** Targeted - inject/purge exist; decoding logs into state updates and the WS ingestion loop - with reorg handling are future phases. + inject/purge exist; the Phase 3 **writer half** (the `StateUpdate` vocabulary + + `apply_update`/`apply_updates`) lands the apply mechanism, but decoding logs + into state updates and the WS ingestion loop with reorg handling are future + phases (Pillar B.2). +- **`inject_v2/v3_*` layer behavior changed in Phase 3 (Decision 2).** The + `protocols`-gated `inject_v2_pool_metadata` / `inject_v3_tick_bitmap*` / + `inject_v3_ticks*` helpers were refolded onto the write-through + `StateUpdate::Slot` primitive, so they now write **both** cache layers (backend + + overlay-if-present) instead of the previous overlay-only write. This is a + deliberate normalization (one consistent write path), not a bug: signatures and + return values are unchanged and the visible reads are identical; only the slot + *placement* across layers moved. `tests/state_update.rs` + (`inject_v3_tick_bitmap_writes_through_to_backend`) pins the new behavior, and + it is recorded in `CHANGELOG.md` (`### Changed`). The cold-backfill + `inject_storage_batch` deliberately remains layer-2-only. - **Recent toolchain.** MSRV 1.88 and edition 2024 are intentional and CI-enforced; consumers on older toolchains are not supported. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 92d6781..e192c15 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -72,7 +72,7 @@ RPC node Event-driven sync ← WS logs · new block | **0** | API hygiene + correctness: drop `amms`, fix `set_block` divergence + `block_in_place` panic, commit the tree. | **Done** (`p0-oss-prep`) | | **1** | Engine seam: typed errors, configurable tx/block env, hot-path benches, builder, `protocols` feature. | **Done** (`phase-1-engine-seam`) | | **2** | Freshness core (Pillar C): `Validity` + `FreshnessRegistry`; observation tracker; policies; optimistic verify-and-rerun loop. | **Done** (`phase-2-freshness`) | -| **3** | State-update primitives (Pillar B.1): `StateUpdate` + targeted writers; refold `inject_*`; surface state-diff output. | Planned | +| **3** | State-update primitives (Pillar B.1): `StateUpdate` + targeted writers; refold `inject_*`; surface state-diff output. | **Done** (`phase-3-state-updates`) | | **4** | Event pipeline + adapters (Pillar B.2): `EventDecoder` trait, V3 adapter, WS ingestion loop, reorg handling. | Planned | | **5** | COW snapshots (Pillar A): structural sharing; overlay buffer reuse. | Planned | @@ -304,6 +304,58 @@ offline `examples/freshness_optimistic.rs`; and `tests/freshness.rs`. --- +## Phase 3 — state-update primitives (detailed, decisions locked) + +Builds **Pillar B.1 — the writer half** of the event → state pipeline: the +generic state-mutation vocabulary and the single apply primitive that writes it +consistently across both cache layers, returning a structured diff. Out of +scope (Phase 4): event decoding (`EventDecoder`, `Log` → `StateUpdate`), the WS +ingestion loop, reorg handling, and overlay-side apply. + +### Locked decisions + +1. **`Account` variant is a partial `AccountPatch`** (`balance`/`nonce`/`code`, + each `Option`), not a full `AccountInfo`: best fit for event-derived writes + (one field at a time) and keeps revm's type out of the public vocabulary. +2. **`inject_v2/v3_*` (`protocols`) normalized to write-through.** Refolded onto + the write-through `StateUpdate::Slot` primitive (backend + overlay-if-present) + instead of the old overlay-only write — a deliberate behavior change recorded + in `CHANGELOG.md` (`### Changed`) and `KNOWN_ISSUES.md`, with a test pinning + the new placement. The cold-backfill `inject_storage_batch` stays layer-2-only. + +### Acceptance — met + +`cargo fmt --check`, `clippy --all-targets -- -D warnings` (default + +`--lib --no-default-features`), `cargo test`, `RUSTDOCFLAGS=-D warnings cargo doc`. + +Landed on `phase-3-state-updates`: `src/state_update.rs` (the generic vocabulary +— `StateUpdate` / `AccountPatch` / `PurgeScope`, the `StateDiff` / `AccountChange` +/ `PurgeRecord` output, reusing `freshness::SlotChange`); `EvmCache::apply_update` +/ `apply_updates` with the dual-layer write-through `Slot`/`Account` and dispatch +`Purge` semantics; the refold of `inject_storage_batch_fresh` / `purge_account` / +`purge_pool_storage` / `purge_pool_slots` / `inject_v2_pool_metadata` / +`inject_v3_*` onto the primitive and the freshness correction-drain routed +through `apply_updates`; the offline `examples/state_update_apply.rs`; +`benches/state_update.rs`; and `tests/state_update.rs`. The §15 addendum adds the +relative / read-modify-write surface — a saturating `SlotDelta`, the +`StateUpdate::SlotDelta` variant, `EvmCache::modify_slot`, and the cold-aware +skip-and-surface contract via the new `StateDiff.skipped` field — to keep +event-derived balances (e.g. ERC-20 `Transfer` deltas) hot without knowing the +resulting absolute value. The §16 post-audit remediation then fixed a +HIGH-severity silent-corruption bug — `cached_storage_value` now mirrors the EVM +`SLOAD` for `StorageCleared`/`NotExisting` overlay accounts instead of returning a +shadowed backend value — and hardened the surface: a no-op `Account` patch no +longer materializes a backend account; the vocabulary and diff gained `serde`; +`StateDiff`/`AccountPatch` became `#[non_exhaustive]`; relative native-balance +tracking landed (`StateUpdate::BalanceDelta`, `EvmCache::modify_account_balance`, +`StateDiff.skipped_balances`, `SkippedBalanceDelta`) with discoverable skip +accessors (`has_skipped`/`skipped_len`/`is_fully_applied`) and the +`StateUpdate::nonce`/`code`/`account` constructors; and `apply_updates` gained a +batched single-lock fast-path (byte-identical to the sequential fold, pinned by an +equivalence test). + +--- + ## Key abstractions for later phases (sketches) ```rust diff --git a/docs/phase-3-spec.md b/docs/phase-3-spec.md new file mode 100644 index 0000000..199eaf5 --- /dev/null +++ b/docs/phase-3-spec.md @@ -0,0 +1,860 @@ +# Phase 3 implementation spec — state-update primitives (Pillar B.1) + +Implementation contract for the **targeted state-mutation vocabulary** and the +single apply primitive that writes it correctly across both cache layers, +returning a structured state diff. Read this **with** +[`ROADMAP.md`](ROADMAP.md) (the "Phase 3" row and the "Pillar B — event → state +pipeline" / "Key abstractions" sections are the design of record). This document +is the precise build contract; where they overlap, prefer this. + +This is **Pillar B.1 — the writer half** of the event → state pipeline. It does +**not** decode events (no `EventDecoder`, no `Log` parsing, no WS loop): that is +Phase 4. Phase 3 builds the vocabulary an event decoder will *emit into* and the +mechanism that *applies* it, with no protocol or event knowledge in the core. + +## 0. Ground rules (non-negotiable) + +- **Branch:** create `phase-3-state-updates` off the current `phase-2-freshness` + HEAD. Commit there in logical steps. Do **not** push, do **not** tag. Commits + must be unsigned: `git -c commit.gpgsign=false commit …` (the 1Password signing + agent is unavailable here). End every commit message with exactly: + `Co-Authored-By: Claude Opus 4.8 (1M context) ` +- **The whole state-update surface is generic core** — it must compile and lint + with `--no-default-features`. `StateUpdate` / `PurgeScope` / `AccountPatch` / + `StateDiff` / `apply_update` / `apply_updates` must NOT depend on the + `protocols` feature. (The *refold* of the `protocols`-gated `inject_v2/v3_*` + helpers stays behind `protocols`, but it consumes the generic primitive.) +- **Green bar at every commit, both feature configs:** + - `cargo fmt --all --check` + - `cargo clippy --all-targets --no-deps -- -D warnings` + - `cargo clippy --lib --no-default-features --no-deps -- -D warnings` + - `cargo test` + - `RUSTDOCFLAGS="-D warnings" cargo doc --no-deps` +- MSRV is 1.88 — no newer-than-1.88 std APIs. Edition 2024. +- **Do not break existing behavior or any existing test.** Existing `inject_*` / + `purge_*` public methods keep their signatures and return values; they become + thin wrappers over the new primitive (the Phase 1 `call_raw` → `call_raw_with` + pattern). The one place a *deliberate* behavior change is on the table is + Decision 2 (§12) — and only with sign-off + a CHANGELOG/KNOWN_ISSUES entry. +- No new dependencies. (`alloy-primitives`, `revm`, `foundry-fork-db` are present.) + +## 1. Objective & scope + +Today the crate writes cached state through a scatter of ad-hoc methods with +**inconsistent layering**: + +| Method | Layer 1 (CacheDB overlay) | Layer 2 (BlockchainDb) | Creates overlay acct? | +| --- | --- | --- | --- | +| `inject_storage_batch` | — | write | no | +| `inject_storage_batch_fresh` | write-through *if present* | write | no | +| `inject_v2_pool_metadata` / `inject_v3_*` | write (via `insert_account_storage`) | — | **yes** | +| `purge_account` | remove acct | remove acct + storage | n/a | +| `purge_pool_storage` | clear storage | remove storage | n/a | +| `purge_pool_slots` | remove slots | remove slots | n/a | +| `override_account_code*` | insert info | insert info | n/a | + +Three different slot-write semantics, no machine-readable record of *what +changed*, and no single vocabulary an event decoder can target. Phase 3 fixes +all three: + +1. **`StateUpdate`** — a small, generic enum: the vocabulary of targeted + mutations (`Slot`, `Account`, `Purge`). This is what a Phase 4 `EventDecoder` + will produce. +2. **`EvmCache::apply_update` / `apply_updates`** — the *single* primitive that + applies a `StateUpdate` (or batch) with **one, documented, consistent** + dual-layer policy, returning a `StateDiff`. +3. **`StateDiff`** — the structured "what actually changed" output (slot/account + diffs + purge records), so callers (and Phase 4's reconciliation) can observe + the effect of an apply. +4. **Refold** the existing `inject_*` / `purge_*` writers onto the primitive so + there is exactly one place the dual-layer write logic lives. + +**In scope:** the generic vocabulary; the apply primitive with write-through +semantics; the state-diff output; refolding the storage-slot and purge writers; +routing the freshness controller's correction drain through the primitive; +offline tests, an example, a benchmark, and docs. + +**Out of scope (document as Phase 4/5 follow-ups, do not build):** +- **Event decoding** — `EventDecoder` trait, V3/V2 adapters, `Log` → `StateUpdate` + (Phase 4). Phase 3 ends at the vocabulary; nothing parses a `Log`. +- **WS ingestion / `on_new_block` apply-and-purge / reorgs / RPC reconciliation** + (Phase 4). +- **Overlay-side apply** — `EvmOverlay::apply` so a live overlay receives updates + mid-fan-out (Phase 4/5). Phase 3 applies to the **`EvmCache`** only. +- **COW snapshots** (Phase 5) — `apply_*` operates on the existing layers. + +## 2. Reuse these existing pieces (do not reinvent) + +- `cache::EvmCache` (`src/cache/mod.rs`): the dual-layer fields + `self.db.cache.accounts` (CacheDB overlay, layer 1) and `self.blockchain_db` + (`accounts()` / `storage()` `RwLock`s, layer 2); the established write-through + pattern in `inject_storage_batch_fresh` (the F1 fix — **the** reference for + correct slot-write layering); `cached_storage_value`; `purge_account` / + `purge_pool_storage` / `purge_pool_slots` (the purge layer logic to fold in); + `self.db.insert_account_info` / `insert_account_storage` (CacheDB writers). +- `freshness::SlotChange { address, slot, old, new }` (`src/freshness.rs`, + re-exported at crate root) — **reuse it** as the slot-diff type; do not define a + parallel one. `StateDiff.slots: Vec`. +- `revm::state::{AccountInfo, Bytecode}` — the account representation in both + layers; `Bytecode::hash_slow()` recomputes a code hash. +- `alloy_primitives::{Address, U256, B256, Bytes}`. +- The offline test/example harness: `tests/common`, `examples/support/mock.rs` + (mocked provider; `from_backend` cache construction with no network). + +## 3. Module layout + +- **`src/state_update.rs`** (new, top-level, generic, **non-`protocols`**): the + pure data types — `StateUpdate`, `PurgeScope`, `AccountPatch`, `StateDiff`, + `AccountChange`, `PurgeRecord` — plus their constructors / small helpers and + in-module unit tests. No `EvmCache` dependency (pure data + logic on itself). +- **`src/cache/mod.rs`**: `EvmCache::apply_update`, `EvmCache::apply_updates`, + and the internal per-variant helpers. Refold `inject_storage_batch_fresh`, + `purge_account`, `purge_pool_storage`, `purge_pool_slots`, + `override_account_code*`, and (Decision 2) `inject_v2/v3_*` onto them. +- **`src/freshness.rs`**: route the `FreshnessController::run` `pending` drain + through `apply_updates` (§9) — behavior-preserving. +- **`src/lib.rs`**: `pub mod state_update;` + re-export + `StateUpdate, PurgeScope, AccountPatch, StateDiff, AccountChange, PurgeRecord`. + +## 4. Types & behavior + +### 4.1 `StateUpdate` — the vocabulary + +```rust +/// A single targeted mutation to cached EVM state. +/// +/// The vocabulary an event decoder (Phase 4) emits and [`EvmCache::apply_update`] +/// consumes. Generic: carries no protocol or event knowledge. +#[derive(Clone, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum StateUpdate { + /// Set one storage slot to `value`, authoritative across both cache layers. + Slot { address: Address, slot: U256, value: U256 }, + /// Patch an account's balance/nonce/code (partial — see [`AccountPatch`]). + Account { address: Address, patch: AccountPatch }, + /// Purge cached state for `address` at `scope`; the next read re-fetches. + Purge { address: Address, scope: PurgeScope }, +} +``` + +Constructors for ergonomics: `StateUpdate::slot(addr, slot, value)`, +`StateUpdate::balance(addr, value)`, `StateUpdate::purge(addr, scope)`. The enum +is `#[non_exhaustive]` (new variants — e.g. a code-only convenience — may be +added pre-1.0 without a breaking change). + +### 4.2 `AccountPatch` — partial account mutation + +```rust +/// A partial account mutation: each `Some` field overwrites the cached value, +/// each `None` leaves it unchanged. Setting `code` recomputes the code hash; +/// `Some(empty bytes)` clears code to the empty-code hash. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct AccountPatch { + pub balance: Option, + pub nonce: Option, + pub code: Option, +} +``` +Builders: `AccountPatch::default()`, `.balance(U256)`, `.nonce(u64)`, `.code(Bytes)` +(each returns `Self`). Rationale for **partial** (vs. a full `AccountInfo`): the +Pillar B driver is events, which usually carry *one* field (a `Transfer` changes +a balance, not nonce/code). Partial application avoids forcing a caller to +reconstruct a full `AccountInfo` (and avoids leaking revm's type into the public +vocabulary). **See Decision 1 (§12).** + +### 4.3 `PurgeScope` + +```rust +/// What part of an address's cached state a purge removes. +#[derive(Clone, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum PurgeScope { + /// Full account: `AccountInfo` (balance/nonce/code) **and** all storage. + /// Equivalent to today's `purge_account`. + Account, + /// All storage slots; account info preserved. Equivalent to `purge_pool_storage`. + AllStorage, + /// Only the listed storage slots. Equivalent to `purge_pool_slots`. + Slots(Vec), +} +``` + +### 4.4 `StateDiff` — the output + +```rust +/// What an `apply_*` call actually changed. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct StateDiff { + /// Storage slots whose value changed (old != new). + pub slots: Vec, // reused from `freshness` + /// Accounts whose balance/nonce/code-hash changed. + pub accounts: Vec, + /// Purges performed, with what they removed. + pub purged: Vec, +} + +/// An account field delta. Each field is `Some((old, new))` only when it changed. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AccountChange { + pub address: Address, + pub balance: Option<(U256, U256)>, + pub nonce: Option<(u64, u64)>, + pub code_hash: Option<(B256, B256)>, +} + +/// Record of a purge: how much of each layer it removed. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PurgeRecord { + pub address: Address, + pub scope: PurgeScope, + /// Storage slots removed from the BlockchainDb backend (layer 2). + pub slots_removed: usize, + /// Whether an `AccountInfo` was removed (only the `Account` scope). + pub account_removed: bool, +} +``` +`StateDiff` helpers: `is_empty()`, `len()` (total changed entries), +`merge(&mut self, other: StateDiff)` (used by `apply_updates` to fold per-update +diffs). **Only actual changes are recorded** — applying a `Slot` whose value +already matches the cache yields an empty diff (idempotence is observable). + +## 5. `EvmCache::apply_update` / `apply_updates` + +```rust +pub fn apply_update(&mut self, update: &StateUpdate) -> StateDiff; +pub fn apply_updates(&mut self, updates: &[StateUpdate]) -> StateDiff; +``` +`apply_updates` folds left, merging each per-update `StateDiff`; later updates +observe the effect of earlier ones (e.g. two `Slot` writes to the same key: the +first records old→a, the second a→b). Both are **synchronous, infallible** +(no RPC — a write primitive, not a fetch). They return the diff; they do not +error. (Account/Slot writes can always succeed against the in-memory layers.) + +### 5.1 `Slot` — write-through (authoritative) + +Identical semantics to `inject_storage_batch_fresh` (the F1-fix reference): +1. `old = self.cached_storage_value(address, slot)` (overlay ▸ backend ▸ `None`). +2. Write `value` into the BlockchainDb backend (layer 2). +3. Write `value` into the CacheDB overlay **iff an overlay account already + exists** for `address` (`self.db.cache.accounts.get_mut`). Do **not** + materialize a new overlay account (preserves the cold-prefetch / layer-2-only + invariant; materializing one could shadow later RPC reads, and a + `StorageCleared` overlay account reads missing slots as ZERO). +4. Record `SlotChange { address, slot, old: old.unwrap_or(ZERO), new: value }` + **only if** `old.unwrap_or(ZERO) != value`. + +> A slot the cache never saw is treated as `old = ZERO` (the value a sim would +> have read), consistent with `verify_slots`. + +### 5.2 `Account` — partial patch, write-through + +1. Load the current `AccountInfo` from the cached layers only (overlay ▸ backend + ▸ `AccountInfo::default()`); remember the `old` field values for the change + record. **No RPC** (apply is a write, not a fetch). +2. Apply each `Some` patch field: `balance`, `nonce`, and for `code` set + `info.code = Some(Bytecode::new_raw(bytes))` (the empty bytecode for empty + input) and `info.code_hash = .hash_slow()`. +3. Write-through, mirroring §5.1: write the patched `AccountInfo` into the + BlockchainDb backend (layer 2) **always**, and into the CacheDB overlay + (`insert_account_info`) **iff an overlay account already exists** (do not + materialize a new overlay account — the read path falls through to the backend + for an absent overlay entry, so a backend-only write is authoritative and we + avoid polluting layer 1). This keeps the winning layer correct without the + cold-backfill hazard. +4. Record an `AccountChange` with `Some((old,new))` only for fields that changed + (compare balance, nonce, code_hash). + +### 5.3 `Purge` — dispatch to existing layer logic + +Dispatch on `scope` to the **existing** purge implementations (now sharing one +home), returning a `PurgeRecord`: +- `Account` → `purge_account` logic: remove from overlay accounts, backend + accounts, backend storage. `account_removed` = removed from any account layer; + `slots_removed` = backend storage slots removed. +- `AllStorage` → `purge_pool_storage` logic (clear overlay storage, remove + backend storage); `slots_removed` = backend slots removed. +- `Slots(slots)` → `purge_pool_slots` logic; `slots_removed` = backend slots + removed. + +## 6. Refold map (existing → primitive) + +Every existing public method **keeps its signature and return value**; it +becomes a wrapper. Existing tests must pass unchanged. + +| Existing | Refold | Public API | +| --- | --- | --- | +| `inject_storage_batch_fresh(&[(a,s,v)])` | `apply_updates` of `Slot`s (discard diff) | unchanged (`-> ()`) | +| `purge_account(a)` | `apply_update(Purge{a, Account})` | unchanged (`-> ()`) | +| `purge_pool_storage(a) -> usize` | `apply_update(Purge{a, AllStorage})`; return `rec.slots_removed` | unchanged | +| `purge_pool_slots(a, slots) -> usize` | `apply_update(Purge{a, Slots(..)})`; return `rec.slots_removed` | unchanged | +| `override_account_code*` | **best-effort**: route its final write through `apply_update(Account{ patch: code })` **only if** behavior-equivalent; it has bespoke target-creation (`MissingTargetBehavior`) + source→target code-copy semantics, so if the refold is not cleanly equivalent, leave the method as-is and only cross-reference the primitive in its doc | unchanged | +| `inject_v2_pool_metadata`, `inject_v3_*` (`protocols`) | build `Vec`, `apply_updates` | **Decision 2 (§12)** | + +**Not refolded (kept distinct, documented):** +- `inject_storage_batch(&[(a,s,v)])` — the **layer-2-only cold-backfill** path + (deliberately no write-through, no overlay touch). This is a *different intent* + from `StateUpdate::Slot` (authoritative write-through). Keep it as the + low-level backfill primitive; add a doc line cross-referencing `apply_update` + for authoritative writes. +- `purge_contracts_storage`, `purge_all_storage` — multi-address / whole-cache + sweeps. Leave as-is (they already share the layer logic); optionally note they + are batch forms of `Purge{AllStorage}`. Not required to refold. + +## 7. Public re-exports + +`src/lib.rs`: `pub mod state_update;` and +```rust +pub use state_update::{ + AccountChange, AccountPatch, PurgeRecord, PurgeScope, StateDiff, StateUpdate, +}; +``` +(`SlotChange` is already re-exported from `freshness`.) + +## 8. `cargo doc` / rustdoc requirements + +- A module-level `//!` doc on `state_update.rs`: the vocabulary, the apply + primitive, the dual-layer write-through policy (one paragraph: backend always, + overlay-if-present, no new overlay account for slots), the `StateDiff` output, + and the **Pillar B.1** framing with an explicit "events are Phase 4" boundary. +- Rustdoc on **every** public item (no `missing_docs` gate, but `-D warnings` + must pass and the surface must be documented thoroughly). +- A short **runnable doctest** on `apply_update` (or the module): build nothing + network-bound — construct `StateUpdate`s and an `AccountPatch`, show the + vocabulary and a `StateDiff` shape. (If a doctest needs an `EvmCache`, gate it + `no_run` and use the example harness pattern; prefer a pure-data doctest.) + +## 9. Freshness integration (behavior-preserving) + +Route `FreshnessController::run`'s `pending` drain (currently +`cache.inject_storage_batch_fresh(&injects)`) through the new primitive: +`cache.apply_updates(&pending.iter().map(|c| StateUpdate::slot(c.address, c.slot, c.new)).collect::>())`. +This is **behavior-identical** (both are write-through), and demonstrates the one +unified write path. Do not change any freshness test expectation. (The validator +itself still flows corrections back as `SlotChange`s; only the main-thread apply +changes its call.) + +## 10. Tests (offline, no network) — authored as the acceptance contract + +These are written **before** implementation and define correctness. Unit tests +in-module (`#[cfg(test)]` in `state_update.rs`); apply/refold integration tests +in a new `tests/state_update.rs` (reuse `tests/common`). + +**`state_update.rs` unit (pure data):** +- `AccountPatch` builders compose; `Default` is all-`None`. +- `StateDiff::merge` concatenates and `is_empty`/`len` count correctly. +- `StateUpdate` constructors produce the expected variants. + +**`tests/state_update.rs` integration (mocked-provider / `from_backend` cache):** +1. **Slot write-through, overlay present:** seed an overlay account + slot; + `apply_update(Slot)`; assert both layers hold the new value and the synchronous + SLOAD path reads it; `StateDiff.slots == [SlotChange{old,new}]`. +2. **Slot write-through, no overlay account:** apply to an address with no overlay + entry; assert the backend holds it, **no overlay account was materialized**, + and a subsequent read sees the value. +3. **Slot no-op:** apply the same value already cached → empty `StateDiff`. +4. **Slot idempotence:** apply twice → first diff non-empty, second empty. +5. **Account balance patch:** patch balance only; assert balance changed, + nonce/code preserved; `AccountChange.balance == Some((old,new))`, + `nonce/code_hash == None`. +6. **Account code patch:** patch code; assert `code_hash` recomputed + (`Bytecode::hash_slow`), `code_hash` delta recorded; balance/nonce preserved. +7. **Account create:** patch an absent account → materialized with patched fields. +8. **Purge Account / AllStorage / Slots:** correct layers cleared; `PurgeRecord` + counts (`slots_removed`, `account_removed`) correct on both layers. +9. **`apply_updates` fold + merge:** a mixed batch (Slot, Account, Purge) → + merged `StateDiff`; later-overrides-earlier ordering for same-key slots. +10. **Refold equivalence:** `purge_pool_storage` wrapper returns the same `usize` + as the pre-refold behavior on a seeded cache; `inject_storage_batch_fresh` + wrapper leaves the cache in the same state as the equivalent `apply_updates`. +11. **(Decision 2, if "normalize"):** `inject_v3_*` now writes through to the + backend (layer 2) — pin the new behavior. + +**Existing suites must stay green** — `tests/freshness.rs`, +`tests/cache_state.rs`, `tests/snapshot_overlay.rs`, the `protocols` cache tests. + +## 11. Docs, example & benchmark + +- **Example** `examples/state_update_apply.rs` (offline, `examples/support`): + build a `from_backend` cache, apply a batch — a `Slot`, an `Account` balance + patch, and a `Purge { Slots }` — then print the returned `StateDiff` (slots + changed, account deltas, purge records). Add a row to the README "Examples" + table (Advanced). +- **Benchmark** `benches/state_update.rs` (offline): `apply_updates` throughput + across batch sizes (1 → 1000), and per-variant cost (Slot vs Account vs Purge), + building the cache once. Register `[[bench]]` in `Cargo.toml` and add a row to + the README "Benchmarks" table. Mirror `benches/freshness.rs` structure. +- **CHANGELOG**: an `### Added` entry for the state-update vocabulary + apply + + diff; if Decision 2 = normalize, a `### Changed` entry for the `inject_v3_*` + layer behavior. +- **ROADMAP**: flip the Phase 3 row to **Done** with the landing branch, mirroring + the Phase 2 "Landed on …" paragraph. +- **KNOWN_ISSUES**: if Decision 2 = normalize, add an entry recording the + `inject_v2/v3_*` layer-behavior change (and that tests now pin it). + +## 12. Decisions (LOCKED) + +> Mirrors the Phase 2 "locked decisions" gate. Both were confirmed with the user +> on 2026-06-15 before the acceptance tests were authored. + +**Decision 1 — `Account` variant shape. → LOCKED: partial `AccountPatch`.** +`Account { address, patch: AccountPatch { balance: Option, nonce: +Option, code: Option } }` (§4.2). Each `Some` overwrites, `None` +leaves as-is. Best fit for event-derived writes (one field at a time); no revm +type leaked into the public vocabulary. (The full-`AccountInfo` alternative from +the ROADMAP sketch is **not** taken.) + +**Decision 2 — `inject_v2/v3_*` refold behavior. → LOCKED: normalize to +write-through.** Refold the `protocols`-gated `inject_v2_pool_metadata` / +`inject_v3_*` helpers onto the write-through `StateUpdate::Slot` primitive +(backend + overlay-if-present) instead of today's layer-1-only write. This is a +deliberate behavior change and **requires**: a CHANGELOG `### Changed` entry, a +KNOWN_ISSUES entry, and test #11 (§10) pinning the new write-through behavior. +The `protocols` pool tests do not pin layer placement, so they stay green. + +## 13. Build order (commit per step, green each time) + +1. `src/state_update.rs`: `StateUpdate`, `PurgeScope`, `AccountPatch`, + `StateDiff`, `AccountChange`, `PurgeRecord` + constructors/helpers + unit + tests; `lib.rs` re-exports. +2. `EvmCache::apply_update` / `apply_updates` (Slot, Account, Purge) + the + `tests/state_update.rs` integration tests. +3. Refold `inject_storage_batch_fresh`, `purge_account`, `purge_pool_storage`, + `purge_pool_slots`, `override_account_code*`, and (per Decision 2) + `inject_v2/v3_*`; route the freshness drain through `apply_updates`. +4. Example + benchmark + README rows. +5. Docs (module `//!`, item rustdoc, doctest), CHANGELOG, ROADMAP → Done, + KNOWN_ISSUES (if normalize). + +## 14. Final acceptance + +Both feature configs green (§0). All new + existing tests pass (`tests/state_update.rs` ++ the in-module unit tests + the untouched existing suites). The example runs +offline and prints a non-trivial `StateDiff`. The benchmark builds and runs. +Report: what landed per file, the public API added, the refold map (with any +behavior change called out), test coverage, and the verification output. + +--- + +## 15. Addendum — relative / read-modify-write updates (decisions LOCKED) + +> Added 2026-06-15 after the §1–§14 surface landed (green, uncommitted). Motivated +> by the event-driven balance-tracking case: a caller indexing ERC-20 `Transfer` +> logs to keep a tracked account's balance hot only learns the **delta** +> (`amount`), not the resulting absolute balance, so the engine must support +> *relative* updates — read the current value, apply a mutation, write back. The +> §4 vocabulary today is **absolute-only**; this addendum adds the relative +> capability. It remains generic core (no protocol knowledge; the slot derivation +> and the ± decision belong to the caller / the Phase-4 decoder). + +### 15.1 The correctness constraint (non-negotiable) + +A relative update is only valid against a value the cache **actually holds**. An +un-fetched ("cold") slot has *no* value — and `cached_storage_value` / `apply_slot` +treat absent as `ZERO`. Applying `delta` to a cold slot would compute +`0 ± amount`, write a wrong value, and (write-through) make it authoritative — +silently corrupting state. Therefore relative application must be **cold-aware**: +apply only when the current value is known; otherwise **skip and surface** it. + +### 15.2 Locked decisions + +**Decision 3 — shape. → LOCKED: vocabulary variant + method.** Add *both*: +(a) a data-level [`StateUpdate::SlotDelta`] variant (so it flows through +`apply_updates` and a Phase-4 `EventDecoder` can emit it as data); (b) a general +`EvmCache::modify_slot` closure escape hatch for arbitrary transforms. + +**Decision 4 — cold-slot handling. → LOCKED: skip & surface.** A `SlotDelta` +targeting a slot absent from **both** layers is **not applied**; it is recorded +in `StateDiff.skipped` so the caller can fetch+seed the true value (the next read +otherwise lazily fetches it). For `modify_slot`, the closure receives +`Option` (`None` when cold) and decides. Overflow is **saturating** +(`Add` clamps at `U256::MAX`, `Sub` at `U256::ZERO`). + +### 15.3 Types (in `src/state_update.rs`) + +```rust +/// A relative storage-slot mutation: read the current value, transform it, write +/// back. Both directions saturate (`Add` at `U256::MAX`, `Sub` at `U256::ZERO`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SlotDelta { Add(U256), Sub(U256) } +impl SlotDelta { + /// Apply the (saturating) delta to a current value. + pub fn apply(self, current: U256) -> U256; +} + +// New variant on the existing enum: +pub enum StateUpdate { + Slot { address, slot, value }, + SlotDelta { address: Address, slot: U256, delta: SlotDelta }, // NEW + Account { address, patch }, + Purge { address, scope }, +} +impl StateUpdate { + /// Construct a relative slot update. + pub fn slot_delta(address: Address, slot: U256, delta: SlotDelta) -> Self; +} + +/// A relative update that could not be applied because the slot's current value +/// is unknown (not cached in either layer). Fetch+seed the slot, then retry. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SkippedDelta { pub address: Address, pub slot: U256, pub delta: SlotDelta } + +// New field on the existing StateDiff (Default = empty): +pub struct StateDiff { + pub slots: Vec, + pub accounts: Vec, + pub purged: Vec, + pub skipped: Vec, // NEW +} +``` +`StateDiff::merge` also extends `skipped`. `is_empty()` / `len()` remain +**changes-only** (slots + accounts + purged) — a skip is *not* a change; document +that `skipped` is separate informational metadata (it does not affect +`is_empty`/`len`, so the §10 no-op/idempotence expectations are unchanged). + +### 15.4 `EvmCache` behavior + +- `apply_update(StateUpdate::SlotDelta { address, slot, delta })`: if + `cached_storage_value(address, slot)` is `Some(current)`, write + `delta.apply(current)` through both layers (reuse the §5.1 write path) and push + a `SlotChange` iff it changed; if `None` (cold), push a `SkippedDelta` to + `diff.skipped` and write nothing. +- `modify_slot(&mut self, address: Address, slot: U256, f: impl FnOnce(Option) -> Option) -> Option`: + call `f` with the current cached value (`None` if cold); if it returns + `Some(new)`, write-through (same path) and return the `SlotChange` iff + `old.unwrap_or(ZERO) != new`; if it returns `None`, write nothing and return + `None`. (The caller owns the cold/overflow policy here; e.g. + `|cur| cur.map(|v| v.saturating_add(amount))` implements skip-on-cold.) +- Refactor the dual-layer slot write out of `apply_slot` into a private + `write_slot_through(address, slot, value)` helper shared by `apply_slot`, + the `SlotDelta` handler, and `modify_slot` (one write path). + +Scope note: account-native-ETH-balance relative updates (an `AccountDelta` / +`modify_account_balance`) are **out of scope** here — the asked case is ERC-20, +whose balances are storage slots. Document that they can be added symmetrically +later if native-ETH tracking is needed. + +### 15.5 Tests (append to `tests/state_update.rs`) + +- `slot_delta_add_applies_to_hot_slot` — seed (backend) 100, `Add(50)` → 150; + `diff.slots == [SlotChange{100,150}]`, `diff.skipped` empty. +- `slot_delta_sub_saturates_at_zero` — seed 30, `Sub(50)` → 0. +- `slot_delta_add_saturates_at_max` — seed `MAX-1`, `Add(10)` → `MAX`. +- `slot_delta_cold_slot_is_skipped_and_surfaced` — fresh (uncached) slot, `Add(50)` + → not applied; `diff.slots` empty; `diff.skipped == [SkippedDelta{..}]`; + `cached_storage_value` still `None`. +- `slot_delta_writes_through_both_layers` — overlay-resident slot (install account + + seed), `Add` updates both overlay and backend. +- `modify_slot_applies_transform` — seed 10, `|c| c.map(|v| v*2)` → 20. +- `modify_slot_closure_skips_cold` — fresh slot, `|c| c.map(|v| v+1)` → returns + `None`, nothing written, slot still cold. +- `modify_slot_can_write_absolute_on_cold` — fresh slot, `|_| Some(7)` → writes 7 + (caller's explicit choice), `SlotChange{0,7}`. +- `state_diff_merge_includes_skipped` — merge concatenates `skipped`. +- `balance_tracking_scenario` — **the motivating end-to-end case**: seed two + holders' balance slots, then apply a `Transfer` as + `[SlotDelta::Sub(amount) on from, SlotDelta::Add(amount) on to]` via + `apply_updates`; assert both balances are correct and `from + to` is conserved. + +### 15.6 Docs / example / changelog + +- Rustdoc on every new item; the module `//!` doc gains a short "relative updates" + paragraph (the cold-aware read-modify-write rule). +- Extend `examples/state_update_apply.rs` (or a focused addition) to show a + `SlotDelta` balance bump **and** a cold-slot skip surfaced via `diff.skipped`. +- Optionally extend `benches/state_update.rs` with a `SlotDelta` apply case + (not required). +- CHANGELOG `### Added`: the relative-update vocabulary (`SlotDelta`, + `StateUpdate::SlotDelta`, `modify_slot`, `StateDiff.skipped`). Note the + `StateDiff` field addition under the pre-1.0 break policy. +- ROADMAP: fold a one-line mention into the Phase 3 "Landed on …" paragraph. + +### 15.7 Acceptance (addendum) + +All of §14 plus: the new tests pass; `diff.skipped` is exercised; the +`balance_tracking_scenario` demonstrates the motivating use case end-to-end; both +feature configs stay green. + +--- + +## 16. Addendum — post-audit remediation (COMPREHENSIVE, decisions LOCKED) + +> Added 2026-06-15 after a 5-lens adversarial audit of the §1–§15 surface (bugs, +> API design, coverage, benchmarks). The user selected the **Comprehensive** +> remediation scope. This section is the precise build contract for that scope. +> Every item below is LOCKED. Where this section conflicts with earlier sections, +> prefer this. Hard rules of §0 still apply (offline tests, both feature configs +> green, MSRV 1.88, edition 2024, no new deps, unsigned commits). + +### 16.0 The correctness bug (P0 — must fix first) + +**Defect (audit HIGH + MED, verified with a reproducer):** the cold-aware safety +guarantee rests on `EvmCache::cached_storage_value` returning what the EVM would +`SLOAD`. That invariant is **false** for an overlay account whose revm +`account_state` is `StorageCleared` or `NotExisting`: for a slot absent from the +overlay storage map, the live `CacheDB::storage`/`storage_ref` returns **ZERO and +never consults the backend**, but `cached_storage_value` (src/cache/mod.rs +~1404-1412) falls through to the BlockchainDb backend and returns +`Some(backend_value)`. Consequences: a `SlotDelta`/`modify_slot` computes +`delta.apply(backend_value)` against a base the EVM never sees (silent +corruption), and `apply_slot` records a wrong `SlotChange.old` and mis-gates the +change predicate. `install_mock_erc20` produces exactly this state +(`replace_account_storage` ⇒ `StorageCleared`), and a backend-only seed via +`inject_storage_batch` is invisible to the EVM — which is why +`balance_tracking_scenario` currently passes while asserting against the buggy +accessor instead of a real `SLOAD`. + +**Fix (LOCKED): make `cached_storage_value` `account_state`-aware**, mirroring +`CacheDB::storage_ref`: +```rust +pub fn cached_storage_value(&self, address: Address, slot: U256) -> Option { + if let Some(db_account) = self.db.cache.accounts.get(&address) { + if let Some(value) = db_account.storage.get(&slot) { + return Some(*value); + } + // Match the EVM SLOAD: a StorageCleared / NotExisting overlay account + // reads a missing slot as ZERO and never consults the backend. + if matches!( + db_account.account_state, + AccountState::StorageCleared | AccountState::NotExisting + ) { + return Some(U256::ZERO); + } + } + let storage = self.blockchain_db.storage().read(); + storage.get(&address).and_then(|s| s.get(&slot).copied()) +} +``` +`AccountState` is revm's enum on `DbAccount` (resolve the exact import path; it is +re-exported from the revm database crate already in use). This single fix repairs +the `SlotDelta`/`modify_slot` base read (HIGH) and `apply_slot`'s `old`/predicate +(MED) at once, and also closes the pre-existing same-root mismatch shared by +`verify_slots` / `inject_storage_batch_fresh`. + +**Tests must validate the EVM SLOAD, not the accessor:** +- **New invariant test** (the red reproducer): with `install_mock_erc20` + + backend-only `inject_storage_batch` seed of slot=100, assert + `cached_storage_value(token, slot) == Some(ZERO)` **and** that it equals what a + real `balance_of`/SLOAD reads (both ZERO). Pre-fix this returns `Some(100)`. +- **Re-point `balance_tracking_scenario`**: seed the holder balance slots in an + **EVM-visible** way (overlay-resident via `db_mut().insert_account_storage`, so + the slots are real to the EVM), apply the `SlotDelta` transfer, and assert the + results via `balance_of` (a real `SLOAD`) in addition to `cached_storage_value`. +- **Present-as-ZERO vs cold**: a slot known to be ZERO (overlay-resident `0`, or a + `StorageCleared` account's absent slot) is **hot** — `SlotDelta::Add(50)` ⇒ 50, + recorded in `diff.slots`, **not** in `diff.skipped`. Cold (no overlay account + **and** no backend value) stays skip-and-surface. Add a test for the hot-zero + case (it is currently the untested seam between Decision-4 skip and apply). + +### 16.1 No-op `Account` patch must not materialize a backend account (audit LOW) + +`apply_account_patch` (src/cache/mod.rs ~1331-1340) writes the patched +`AccountInfo` into the backend **unconditionally**, so an all-`None` (or +otherwise no-change) patch on an address absent from both layers inserts +`AccountInfo::default()` into the shared backend map while returning an **empty** +diff — breaking no-op parity with the Slot path and (per the cold-account hazard) +masking a future RPC fetch. **Fix (LOCKED):** compute the change first; **only +write-through when at least one field actually changes** (i.e. skip both layer +writes and return `None` when the patched `info` equals the loaded base). A real +field change on an absent address still materializes the backend account (the +existing intended behavior — keep `apply_account_patch_materializes_absent_account` +green). Add a no-op idempotence test (patching balance to its current value ⇒ +empty diff, no backend account materialized). + +### 16.2 Cold absolute-`Account`-patch hazard — document (audit LOW) + +A *partial* absolute `Account` patch on a cold (un-fetched) address writes default +nonce/code through the shared backend, masking the real on-chain account. This is +spec-locked §5.2 behavior, **not** changed here, but it is an undocumented +live-fork footgun. **Fix (LOCKED, docs only):** add a `### Known issues` entry in +`docs/KNOWN_ISSUES.md` and a prominent `# Warning` doc paragraph on +`apply_update` / `StateUpdate::Account` / `AccountPatch` stating that a partial +patch on an address absent from both cache layers writes default nonce/code as +authoritative (pre-empting RPC), so callers must fetch+seed the account first, or +use `StateUpdate::BalanceDelta` (§16.5) for relative native-balance tracking. + +### 16.3 `serde` on the vocabulary (audit HIGH gap) + +`serde` is a non-optional crate dependency and other public types +(`StorageAccessList`, `PrefetchRegistry`) already derive/serialize. The event +pipeline (the stated motivation) needs to serialize `StateUpdate`s and ship +`StateDiff`s. **Fix (LOCKED):** derive `serde::Serialize, serde::Deserialize` +**unconditionally** on `SlotDelta`, `StateUpdate`, `AccountPatch`, `PurgeScope`, +`StateDiff`, `AccountChange`, `PurgeRecord`, `SkippedDelta`, the new `BalanceDelta` +payload / `SkippedBalanceDelta` (§16.5), **and** `SlotChange` (src/freshness.rs). +Add a JSON round-trip test for a representative `StateUpdate` set and a `StateDiff`. +(All fields are `Address`/`U256`/`B256`/`Bytes`/`u64`/`usize`/`bool` — derives +compile today with the alloy serde features already enabled.) + +### 16.4 `#[non_exhaustive]` on output/record types (audit HIGH) + +`StateDiff` just grew a field as a documented pre-1.0 break, and §16.5 adds +another (`skipped_balances`). **Fix (LOCKED, scoped):** add `#[non_exhaustive]` +to **`StateDiff`** (the aggregate that demonstrably grows) and **`AccountPatch`** +(builder-constructed via `.balance()/.nonce()/.code()` + `Default`). Both are +still constructed by external callers/tests through `Default` + field-assignment +(`StateDiff`) or the builders (`AccountPatch`), so future field additions are +non-breaking at zero ergonomic cost. (`StateUpdate` and `PurgeScope` already are +`#[non_exhaustive]`.) + +**Deliberately NOT `#[non_exhaustive]`** — the leaf record types `SlotChange`, +`AccountChange`, `PurgeRecord`, `SkippedDelta`, and `SkippedBalanceDelta`. These +are routinely **constructed as struct literals in equality assertions** by both +the test suite (`diff.skipped == vec![SkippedDelta { .. }]`, +`diff.slots == vec![SlotChange { .. }]`) and downstream users testing against a +returned diff. `#[non_exhaustive]` would forbid that external construction — a +real, non-zero cost that outweighs the speculative benefit of these stable, +fully-determined shapes gaining a field. (This is a deliberate, documented +departure from the audit finding, which assumed these were read-only; assertion +construction is the counter-case.) + +### 16.5 New capability — account-native-balance delta (audit MED gap) + +Relative-update symmetry: `SlotDelta` covers ERC-20 (storage) balances, but +native-ETH tracking (value transfers, coinbase, selfdestruct) is learned as a +delta too. **Add (LOCKED):** +```rust +// reuse SlotDelta (Add/Sub, saturating) for the relative amount +pub enum StateUpdate { /* … */ BalanceDelta { address: Address, delta: SlotDelta } } // NEW variant +impl StateUpdate { pub fn balance_delta(address: Address, delta: SlotDelta) -> Self; } // NEW ctor + +#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[non_exhaustive] +pub struct SkippedBalanceDelta { pub address: Address, pub delta: SlotDelta } // NEW + +pub struct StateDiff { /* … */ pub skipped_balances: Vec } // NEW field + +impl EvmCache { + /// Read-modify-write the native balance. `f` gets the current cached balance + /// (`None` if the account is absent from both layers); `Some(new)` writes it + /// through (preserving nonce/code), `None` writes nothing. + pub fn modify_account_balance( + &mut self, address: Address, f: impl FnOnce(Option) -> Option, + ) -> Option; +} +``` +- **Cold-aware:** "cold" for a balance = the account is absent from **both** layers + (balance unknown). `account_state` does **not** matter here (it governs storage, + not the basic `AccountInfo`). A `BalanceDelta` on a cold account is **not + applied**; it is surfaced in `StateDiff.skipped_balances` (avoids the §16.2 + masking — we never write a default account). On a present account, load the full + `AccountInfo` (overlay ▸ backend), apply the saturating delta to `info.balance`, + preserve nonce/code, write-through (backend always, overlay-if-present), record + an `AccountChange` (balance only) iff it changed. +- `modify_account_balance` is the closure analog (same load/cold rules; `f` decides). +- `StateDiff::merge` extends `skipped_balances`. `is_empty`/`len` stay + **changes-only**. `has_skipped`/`skipped_len`/`is_fully_applied` (§16.6) count + **both** `skipped` and `skipped_balances`. +- Tests: hot apply (Add/Sub/saturation, AccountChange recorded, nonce/code + preserved), cold skip-and-surface (`skipped_balances` populated, no backend + account materialized), `modify_account_balance` hot/cold/`None`. + +### 16.6 Discoverable skip accessors + loud docs (audit MED footgun) + +A cold-skipped relative update is invisible to the natural `is_empty()`/`len()` +success check, so a dropped balance update can break conservation silently. +**Add (LOCKED)** on `StateDiff`: +- `has_skipped(&self) -> bool` — `!skipped.is_empty() || !skipped_balances.is_empty()`. +- `skipped_len(&self) -> usize` — `skipped.len() + skipped_balances.len()`. +- `is_fully_applied(&self) -> bool` — `!self.has_skipped()`. + +Document prominently on `apply_update`/`apply_updates` that after relative +updates the caller **must** check `has_skipped()`/`skipped`/`skipped_balances` — a +cold target is dropped, not applied. Mirror this in the example. + +### 16.7 Constructor symmetry (audit LOW) + +Add convenience constructors for parity with `slot`/`balance`/`purge`/`slot_delta`: +`StateUpdate::nonce(address, u64)`, `StateUpdate::code(address, Bytes)`, +`StateUpdate::account(address, AccountPatch)`. + +### 16.8 Coverage gaps (audit — all listed) + +Add tests (in `tests/state_update.rs` unless noted). Each must assert the +**layer-correct** outcome, not just the accessor: +- **Account-patch backend-write-always:** on an overlay-present account, assert + `backend_balance(...)` updates (not only overlay). +- **Account-patch no-overlay-materialization:** after patching a backend-only / + absent account that *does* change, assert no *new* overlay account is + materialized where the spec says none should be (mirror + `apply_slot_no_overlay_account_is_not_materialized`). +- **Backend-only account patch:** seed an account only in the backend + (`blockchain_db().accounts().write().insert`), patch balance, assert + `AccountChange.balance == Some((old,new))`, backend updated, overlay still absent. +- **Nonce-only** and **multi-field (balance+nonce+code)** patches: assert the + respective `AccountChange` fields are `Some`/`None` correctly. +- **Empty-code clear:** patch `code(Bytes::new())` over non-empty code ⇒ + `code_hash` → `KECCAK_EMPTY`; patching empty over already-empty ⇒ `None`. +- **Account-patch idempotence/no-op** (also pins §16.1): balance→current ⇒ empty + diff, no backend materialization. +- **Decision-2 pins:** `inject_v2_pool_metadata_writes_through_to_backend` and + `inject_v3_ticks_writes_through_to_backend` (protocols-gated), mirroring the + existing bitmap test. +- **Purge edges:** purge of an absent address ⇒ `PurgeRecord{account_removed:false, + slots_removed:0}`; `PurgeScope::Slots` with some slots absent ⇒ `slots_removed` + counts only present backend slots; overlay-vs-backend accounting (overlay-only + slots not counted in `slots_removed`). +- **`modify_slot` write-through layers:** overlay-present ⇒ both layers updated; + absent ⇒ backend only, no overlay materialized. +- **Batched == sequential equivalence** (the perf-fast-path safety net, §16.9): + a mixed `apply_updates([...])` batch (distinct addresses, a same-address repeat, + and a `Purge` mid-batch) leaves **byte-identical** layer state **and** an + equivalent merged `StateDiff` to applying each update via `apply_update` in + sequence. This test must pass both before and after the perf work. + +### 16.9 Performance (audit — benchmarks) + +Benchmarks showed `apply_updates` ≈ 4.4× the per-element cost of raw +`inject_storage_batch`, dominated by **per-update `RwLock` churn** (a read lock for +the old value + a separate write lock per slot) and a redundant `SlotDelta` read. +This matters because `inject_storage_batch_fresh` and the `inject_v3_*` writers now +route through `apply_updates` for **bulk** seeding. **Fix (LOCKED):** +1. **Eliminate the `SlotDelta` double read:** the `SlotDelta` arm already reads + `cached_storage_value` for `current`; build the `SlotChange` from that value and + call the shared write path directly instead of routing through `apply_slot` + (which re-reads the same slot). +2. **Batched single-lock fast-path** for `apply_updates`: process consecutive + `Slot`/`SlotDelta` writes holding the backend storage write-guard **once** for + the run (overlay access is lock-free on `self.db.cache.accounts`). Preserve + apply order: when an `Account`/`Purge` update is reached, **drop the guard + first** (those take `accounts()` / `storage()` locks themselves — holding the + storage write-guard across `apply_purge` would deadlock on the non-reentrant + `RwLock`), process it, then lazily re-acquire on the next slot run. Correctness + is pinned by the §16.8 batched==sequential equivalence test and the existing + refold-equivalence tests; **do not** weaken any of them. The old-value read must + stay `account_state`-aware (§16.0) even inside the held guard. +3. Single-update `apply_update`/`apply_slot` may keep the read-then-write split + (correctness first); the batch path is where the lock win is realized. + +### 16.10 Missing benchmarks (audit) + +Extend `benches/state_update.rs` (keep existing cases): `SlotDelta` hot-apply and +cold-skip; `modify_slot`; a **heterogeneous** `apply_updates` batch (Slot + +Account + Purge); `Account` **code** patch (the `Bytecode::new_raw` + `hash_slow` +keccak — likely the most expensive single apply); `PurgeScope::Account` and +`PurgeScope::Slots`; a **distinct-address** `apply_updates` batch (the only fair +apples-to-apples vs the `inject_storage_batch` baseline). All benches stay offline +and must build under `cargo bench --no-run`. + +### 16.11 Docs / CHANGELOG / ROADMAP + +- Rustdoc on every new item; update the `state_update` module `//!` doc to cover + `BalanceDelta`, the skip accessors, and the cold-account warning. +- `examples/state_update_apply.rs`: add a `BalanceDelta` bump + a cold + `BalanceDelta` surfaced via `diff.skipped_balances`, and use `has_skipped()`. +- CHANGELOG: `### Fixed` (the `cached_storage_value` corruption bug; the no-op + Account materialization) and `### Added` (`serde`; `#[non_exhaustive]`; + `BalanceDelta`/`modify_account_balance`/`SkippedBalanceDelta`/ + `StateDiff.skipped_balances`; `has_skipped`/`skipped_len`/`is_fully_applied`; + `StateUpdate::nonce`/`code`/`account`). Note the additive `StateDiff` field and + the `#[non_exhaustive]` additions under the pre-1.0 break policy. +- ROADMAP: extend the Phase 3 "Landed on …" paragraph with the §16 remediation. +- KNOWN_ISSUES: the §16.2 cold-account-patch entry. + +### 16.12 Acceptance (remediation) + +All of §14 plus: every §16 test passes; the corruption reproducer is **red before / +green after** the §16.0 fix; `balance_tracking_scenario` validates via a real +`SLOAD`; the batched==sequential equivalence test passes; `serde` round-trips; +both feature configs green (`cargo test`, `clippy` default + `--no-default-features`, +`fmt`, `RUSTDOCFLAGS=-D warnings doc`); `cargo bench --no-run` builds all benches; +the example runs offline and shows a skipped relative update via `has_skipped()`. diff --git a/examples/state_update_apply.rs b/examples/state_update_apply.rs new file mode 100644 index 0000000..fa06944 --- /dev/null +++ b/examples/state_update_apply.rs @@ -0,0 +1,183 @@ +//! Apply a batch of targeted [`StateUpdate`]s and inspect the returned +//! [`StateDiff`] (Phase 3, Pillar B.1) — fully offline. +//! +//! Builds a mocked-provider cache, seeds a little state, then applies a mixed +//! batch — a `Slot` write, an `Account` balance patch, and a `Purge { Slots }` — +//! through the single [`EvmCache::apply_update`] / `apply_updates` primitive, and +//! prints what each apply actually changed (slot deltas, account deltas, purge +//! records). It then shows a *relative* `SlotDelta` balance bump on a hot slot and +//! a cold-slot `SlotDelta` surfaced (not applied) via `diff.skipped`. No network +//! is touched. +//! +//! Run with: +//! +//! ```sh +//! cargo run --example state_update_apply +//! ``` + +use alloy_primitives::{Address, U256}; +use anyhow::Result; +use evm_fork_cache::{PurgeScope, SlotDelta, StateUpdate}; + +#[path = "support/mock.rs"] +mod mock; + +use mock::{install_default_account, install_mock_erc20, offline_cache}; + +#[tokio::main(flavor = "multi_thread")] +async fn main() -> Result<()> { + let pool = Address::repeat_byte(0x11); + let holder = Address::repeat_byte(0x22); + + let mut cache = offline_cache().await?; + // A token-like account with overlay storage (so the slot write heals both + // layers) plus an EOA-style account to patch a balance onto. + install_mock_erc20(&mut cache, pool); + install_default_account(&mut cache, holder); + + // Seed some backend storage on the pool so the purge has something to remove + // and the slot write has a recorded `old` value. + cache.inject_storage_batch(&[ + (pool, U256::from(0), U256::from(100)), // e.g. a reserve slot + (pool, U256::from(7), U256::from(1)), // a tick/aux slot we'll purge + (pool, U256::from(8), U256::from(2)), // another slot we'll purge + ]); + + println!("Applying a mixed batch of state updates...\n"); + + let diff = cache.apply_updates(&[ + // 1. Authoritative slot write (e.g. an event-derived reserve update). + StateUpdate::slot(pool, U256::from(0), U256::from(250)), + // 2. Partial account patch: set only the balance, leave nonce/code. + StateUpdate::balance(holder, U256::from(1_000_000)), + // 3. Drop two stale storage slots so the next read re-fetches them. + StateUpdate::purge(pool, PurgeScope::Slots(vec![U256::from(7), U256::from(8)])), + ]); + + println!("StateDiff: {} changed entr(ies)\n", diff.len()); + + println!("Slot changes ({}):", diff.slots.len()); + for change in &diff.slots { + println!( + " {} slot {} : {} -> {}", + change.address, change.slot, change.old, change.new + ); + } + + println!("\nAccount changes ({}):", diff.accounts.len()); + for change in &diff.accounts { + println!(" {}", change.address); + if let Some((old, new)) = change.balance { + println!(" balance: {old} -> {new}"); + } + if let Some((old, new)) = change.nonce { + println!(" nonce: {old} -> {new}"); + } + if let Some((old, new)) = change.code_hash { + println!(" code: {old} -> {new}"); + } + } + + println!("\nPurge records ({}):", diff.purged.len()); + for rec in &diff.purged { + println!( + " {} scope={:?} slots_removed={} account_removed={}", + rec.address, rec.scope, rec.slots_removed, rec.account_removed + ); + } + + // Re-applying the same slot value is a no-op — idempotence is observable. + let again = cache.apply_update(&StateUpdate::slot(pool, U256::from(0), U256::from(250))); + println!( + "\nRe-applying the same slot value -> empty diff: {}", + again.is_empty() + ); + + // --- Relative (read-modify-write) updates ------------------------------- + // + // A caller indexing ERC-20 `Transfer` logs only learns the *delta* + // (`amount`), not the resulting balance. `SlotDelta` reads the current value + // and applies a saturating mutation, write-through. + println!("\n--- Relative SlotDelta updates ---"); + + // A hot (seeded) balance slot: +750 relative to the current value. + let hot_slot = U256::from(0); // we set this to 250 above + let rel = cache.apply_update(&StateUpdate::slot_delta( + pool, + hot_slot, + SlotDelta::Add(U256::from(750)), + )); + for change in &rel.slots { + println!( + " hot : slot {} {} -> {} (Add 750)", + change.slot, change.old, change.new + ); + } + + // A cold slot the cache never fetched: applying `0 ± amount` would corrupt an + // unknown value, so the delta is NOT applied — it is surfaced for the caller + // to fetch+seed the true value and retry. + let cold_slot = U256::from(4_242); + let cold = cache.apply_update(&StateUpdate::slot_delta( + pool, + cold_slot, + SlotDelta::Add(U256::from(100)), + )); + println!( + " cold : applied {} change(s), skipped {} (left for the caller to seed)", + cold.slots.len(), + cold.skipped.len() + ); + for skip in &cold.skipped { + println!( + " skipped: {} slot {} delta={:?}", + skip.address, skip.slot, skip.delta + ); + } + + // --- Relative native-balance updates (BalanceDelta) --------------------- + // + // The same cold-aware read-modify-write rule applies to an account's native + // ETH balance: a `BalanceDelta` on a *present* account bumps its balance; on a + // *cold* account (absent from both layers) it is dropped and surfaced. + println!("\n--- Relative BalanceDelta updates ---"); + + // `holder` was installed above (present), so a +500_000 delta applies. + let bal = cache.apply_update(&StateUpdate::balance_delta( + holder, + SlotDelta::Add(U256::from(500_000)), + )); + for change in &bal.accounts { + if let Some((old, new)) = change.balance { + println!( + " hot : {} balance {} -> {} (Add 500_000)", + change.address, old, new + ); + } + } + + // A cold account the cache never loaded: the balance is unknown, so the delta + // is NOT applied (no default account is materialized to mask the real one) — + // it is surfaced in `diff.skipped_balances`. + let unknown = Address::repeat_byte(0x99); + let cold_bal = cache.apply_update(&StateUpdate::balance_delta( + unknown, + SlotDelta::Add(U256::from(1_000)), + )); + // A cold-skipped relative update produces no change, so it is invisible to the + // changes-only `is_empty()`/`len()` check — callers MUST inspect `has_skipped()`. + println!( + " cold : has_skipped={} skipped_len={} (changes-only len={})", + cold_bal.has_skipped(), + cold_bal.skipped_len(), + cold_bal.len(), + ); + for skip in &cold_bal.skipped_balances { + println!( + " skipped balance: {} delta={:?}", + skip.address, skip.delta + ); + } + + Ok(()) +} diff --git a/src/cache/mod.rs b/src/cache/mod.rs index b83a640..bb94329 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -56,7 +56,7 @@ use revm::{ Context, ExecuteCommitEvm, ExecuteEvm, InspectEvm, MainBuilder, MainContext, context::{BlockEnv, CfgEnv, Journal, LocalContext, TxEnv, result::ExecutionResult}, context_interface::JournalTr, - database::CacheDB, + database::{AccountState, CacheDB}, primitives::hardfork::SpecId, state::{AccountInfo, Bytecode}, }; @@ -66,6 +66,10 @@ use crate::access_set::StorageAccessList; use crate::errors::{SimError, SimulationError, SimulationResult}; use crate::freshness::SlotChange; use crate::inspector::TransferInspector; +use crate::state_update::{ + AccountChange, AccountPatch, PurgeRecord, PurgeScope, SkippedBalanceDelta, SkippedDelta, + SlotDelta, StateDiff, StateUpdate, +}; use bytecode::BytecodeCache; #[cfg(feature = "protocols")] @@ -128,6 +132,57 @@ fn block_in_place_handle() -> Result { } } +/// Read a storage slot from already-borrowed layers (`account_state`-aware), +/// mirroring [`EvmCache::cached_storage_value`] but operating on a held backend +/// storage guard rather than re-locking. Shared by the batched slot-run fast-path +/// ([`EvmCache::apply_slot_run`]) so the same EVM-SLOAD semantics hold inside the +/// held guard: the overlay slot wins; a `StorageCleared`/`NotExisting` overlay +/// account reads a missing slot as ZERO (the backend is **not** consulted); +/// otherwise it falls through to the backend. +fn read_slot_account_state_aware( + overlay: &std::collections::HashMap, + storage: &std::collections::HashMap, + address: Address, + slot: U256, +) -> Option +where + S1: std::hash::BuildHasher, + S2: std::hash::BuildHasher, +{ + if let Some(db_account) = overlay.get(&address) { + if let Some(value) = db_account.storage.get(&slot) { + return Some(*value); + } + if matches!( + db_account.account_state, + AccountState::StorageCleared | AccountState::NotExisting + ) { + return Some(U256::ZERO); + } + } + storage.get(&address).and_then(|s| s.get(&slot).copied()) +} + +/// Write a storage slot into already-borrowed layers, mirroring +/// [`EvmCache::write_slot_through`] but operating on a held backend storage guard. +/// Backend (layer 2) is always written; the overlay (layer 1) is written only if +/// an overlay account already exists (never materialize a new overlay account). +fn write_slot_into( + overlay: &mut std::collections::HashMap, + storage: &mut std::collections::HashMap, + address: Address, + slot: U256, + value: U256, +) where + S1: std::hash::BuildHasher, + S2: std::hash::BuildHasher + Default, +{ + storage.entry(address).or_default().insert(slot, value); + if let Some(db_account) = overlay.get_mut(&address) { + db_account.storage.insert(slot, value); + } +} + static CACHE_SPEED_MODE: AtomicU8 = AtomicU8::new(CacheSpeedMode::Slow as u8); /// Runtime tuning profile for cache-side batch storage fetches. @@ -1100,17 +1155,538 @@ impl EvmCache { /// the backend write is already authoritative and materializing an overlay /// entry would pollute layer 1 and could shadow later RPC reads. pub fn inject_storage_batch_fresh(&mut self, results: &[(Address, U256, U256)]) { + // Thin wrapper over the unified write primitive (the F1 fix now lives in + // `apply_slot`). Each tuple becomes a write-through `StateUpdate::Slot`; + // the returned diff is discarded to preserve this method's `-> ()` API. + let updates: Vec = results + .iter() + .map(|&(addr, slot, value)| StateUpdate::slot(addr, slot, value)) + .collect(); + let _ = self.apply_updates(&updates); + } + + /// Apply a single targeted [`StateUpdate`], returning a [`StateDiff`] of what + /// actually changed. + /// + /// This is the single primitive that writes the state-update vocabulary + /// across both cache layers with one consistent, documented policy. It is + /// **synchronous and infallible** — a write, not a fetch, so it never touches + /// RPC and never errors. See the [`state_update`](crate::state_update) module + /// for the dual-layer write-through policy and the diff semantics. + /// + /// - [`StateUpdate::Slot`] — write `value` into the backend (layer 2) always, + /// and into the overlay (layer 1) only if an overlay account already + /// exists. Records a [`SlotChange`] only when the value actually changes + /// (`old.unwrap_or(ZERO) != value`). + /// - [`StateUpdate::SlotDelta`] — *relative*, cold-aware. If the slot has a + /// cached value, write the saturating delta through the same path and record + /// a [`SlotChange`] iff it changed; if the slot is cold (absent from both + /// layers), apply nothing and surface a `SkippedDelta` in `diff.skipped`. + /// - [`StateUpdate::BalanceDelta`] — *relative*, cold-aware native-balance + /// update. If the account is present in either layer, apply the saturating + /// delta to its balance (nonce/code preserved) write-through and record an + /// [`AccountChange`] iff it changed; if the account is cold (absent from both + /// layers), apply nothing and surface a [`SkippedBalanceDelta`] in + /// `diff.skipped_balances` (no default account is materialized). + /// - [`StateUpdate::Account`] — load the current `AccountInfo` from the cached + /// layers (no RPC), apply each `Some` patch field (recomputing the code hash + /// when `code` is set), then write through with the same layer policy. + /// Records an [`AccountChange`] with `Some((old, new))` only for fields + /// that changed. + /// - [`StateUpdate::Purge`] — dispatch to the matching purge layer logic and + /// record a [`PurgeRecord`]. + /// + /// # Warning — relative updates can be skipped + /// + /// A relative [`SlotDelta`](StateUpdate::SlotDelta) / + /// [`BalanceDelta`](StateUpdate::BalanceDelta) targeting a **cold** address is + /// *dropped, not applied* (applying it against an unknown base would corrupt + /// state). Because a skip produces no change, it is invisible to the + /// changes-only [`StateDiff::is_empty`] / [`StateDiff::len`] success check, so + /// after applying relative updates the caller **must** inspect + /// [`StateDiff::has_skipped`] (or `diff.skipped` / `diff.skipped_balances`) and + /// fetch+seed the cold target — a silently-dropped balance update can break + /// conservation. + /// + /// # Warning — cold absolute `Account` patches + /// + /// A partial absolute [`StateUpdate::Account`] patch on an address absent from + /// both layers writes default nonce/code through the backend as authoritative, + /// masking a real RPC fetch. Fetch+seed the account first, or use + /// [`StateUpdate::BalanceDelta`] for relative native-balance tracking. + /// + /// ```no_run + /// # use alloy_primitives::{Address, U256}; + /// # use evm_fork_cache::StateUpdate; + /// # fn example(cache: &mut evm_fork_cache::cache::EvmCache) { + /// let pool = Address::repeat_byte(0x01); + /// let diff = cache.apply_update(&StateUpdate::slot(pool, U256::from(0), U256::from(42))); + /// assert_eq!(diff.slots.len(), 1); + /// # } + /// ``` + pub fn apply_update(&mut self, update: &StateUpdate) -> StateDiff { + let mut diff = StateDiff::default(); + match update { + StateUpdate::Slot { + address, + slot, + value, + } => { + if let Some(change) = self.apply_slot(*address, *slot, *value) { + diff.slots.push(change); + } + } + StateUpdate::SlotDelta { + address, + slot, + delta, + } => match self.cached_storage_value(*address, *slot) { + // Hot slot: apply the saturating delta write-through. Build the + // change from the value we already read (do not route through + // `apply_slot`, which would re-read the same slot — §16.9.1). + Some(current) => { + let new = delta.apply(current); + self.write_slot_through(*address, *slot, new); + if current != new { + diff.slots.push(SlotChange { + address: *address, + slot: *slot, + old: current, + new, + }); + } + } + // Cold slot: applying `0 ± amount` would corrupt an unknown value, + // so write nothing and surface the skip for the caller to seed. + None => diff.skipped.push(SkippedDelta { + address: *address, + slot: *slot, + delta: *delta, + }), + }, + StateUpdate::BalanceDelta { address, delta } => { + match self.apply_balance_delta(*address, *delta) { + // Hot account: the saturating delta was applied. + Ok(Some(change)) => diff.accounts.push(change), + // Hot account but no change (e.g. Sub from 0, Add of 0). + Ok(None) => {} + // Cold account: surface the skip; nothing was materialized. + Err(skipped) => diff.skipped_balances.push(skipped), + } + } + StateUpdate::Account { address, patch } => { + if let Some(change) = self.apply_account_patch(*address, patch) { + diff.accounts.push(change); + } + } + StateUpdate::Purge { address, scope } => { + diff.purged.push(self.apply_purge(*address, scope)); + } + } + diff + } + + /// Apply a batch of [`StateUpdate`]s left-to-right, merging each per-update + /// [`StateDiff`]. + /// + /// Later updates observe the effect of earlier ones: two `Slot` writes to the + /// same key record `old → a` then `a → b`. Like + /// [`apply_update`](Self::apply_update) this is synchronous and infallible. + /// + /// # Performance — batched single-lock fast-path + /// + /// Consecutive `Slot`/`SlotDelta` writes are processed holding the backend + /// storage write-guard **once** for the run (the overlay map is lock-free), so + /// a bulk slot seed pays one lock acquisition instead of one read + one write + /// lock per slot. Apply order is preserved: when an `Account`/`BalanceDelta`/ + /// `Purge` update is reached the guard is dropped first (those take the + /// `accounts()` / `storage()` locks themselves — holding the storage + /// write-guard across them would deadlock the non-reentrant `RwLock`), the + /// update is processed via [`apply_update`](Self::apply_update), then the guard + /// is lazily re-acquired on the next slot run. The result is byte-identical to + /// folding [`apply_update`](Self::apply_update) over the batch. + /// + /// # Warning — relative updates can be skipped + /// + /// See [`apply_update`](Self::apply_update): a cold relative update is dropped, + /// not applied, and is invisible to [`StateDiff::is_empty`] / + /// [`StateDiff::len`]. After a batch with relative updates, check + /// [`StateDiff::has_skipped`]. + pub fn apply_updates(&mut self, updates: &[StateUpdate]) -> StateDiff { + let mut diff = StateDiff::default(); + let mut i = 0; + while i < updates.len() { + match &updates[i] { + // A run of consecutive slot writes: process them under a single + // held storage write-guard, then advance past the run. + StateUpdate::Slot { .. } | StateUpdate::SlotDelta { .. } => { + let run_end = updates[i..] + .iter() + .position(|u| { + !matches!(u, StateUpdate::Slot { .. } | StateUpdate::SlotDelta { .. }) + }) + .map(|off| i + off) + .unwrap_or(updates.len()); + self.apply_slot_run(&updates[i..run_end], &mut diff); + i = run_end; + } + // Account / BalanceDelta / Purge: no held guard (they take their + // own locks), so route through the single-update primitive. + _ => { + diff.merge(self.apply_update(&updates[i])); + i += 1; + } + } + } + diff + } + + /// Apply a run of consecutive `Slot`/`SlotDelta` updates under one held backend + /// storage write-guard (§16.9.2), merging each change into `diff`. + /// + /// The backend storage guard is acquired once for the whole run; overlay access + /// is lock-free (`self.db.cache.accounts`). The old-value read stays + /// `account_state`-aware (matching [`cached_storage_value`](Self::cached_storage_value)): + /// for an overlay account whose slot is absent, a `StorageCleared`/`NotExisting` + /// state reads ZERO and the backend is **not** consulted. Behavior is identical + /// to applying each update via [`apply_update`](Self::apply_update); the + /// `apply_updates_batched_equals_sequential` test pins this. + fn apply_slot_run(&mut self, run: &[StateUpdate], diff: &mut StateDiff) { + // Borrow the two layers as disjoint fields: the backend storage guard + // (layer 2) held for the whole run, and the overlay accounts map (layer 1, + // lock-free). + let overlay = &mut self.db.cache.accounts; + let mut storage = self.blockchain_db.storage().write(); + + for update in run { + // Resolve `(address, slot, old, new)` for the write; a cold SlotDelta + // is skipped here (write nothing). `old` is the `account_state`-aware + // read (overlay ▸ cleared-as-ZERO ▸ backend), reused for both the write + // gate and the change record so each slot is read at most once. + let (address, slot, old, new) = match update { + StateUpdate::Slot { + address, + slot, + value, + } => { + let old = read_slot_account_state_aware(overlay, &storage, *address, *slot) + .unwrap_or(U256::ZERO); + (*address, *slot, old, *value) + } + StateUpdate::SlotDelta { + address, + slot, + delta, + } => match read_slot_account_state_aware(overlay, &storage, *address, *slot) { + // Hot: apply the saturating delta to the value already read. + Some(current) => (*address, *slot, current, delta.apply(current)), + // Cold: skip and surface (write nothing). + None => { + diff.skipped.push(SkippedDelta { + address: *address, + slot: *slot, + delta: *delta, + }); + continue; + } + }, + // The caller only ever hands this method slot updates. + _ => unreachable!("apply_slot_run only processes Slot/SlotDelta"), + }; + + write_slot_into(overlay, &mut storage, address, slot, new); + if old != new { + diff.slots.push(SlotChange { + address, + slot, + old, + new, + }); + } + } + } + + /// Write-through a single storage slot (§5.1). Returns a [`SlotChange`] iff + /// the slot's value actually changes. + fn apply_slot(&mut self, address: Address, slot: U256, value: U256) -> Option { + // Old value: overlay ▸ backend ▸ None (treated as ZERO). + let old = self + .cached_storage_value(address, slot) + .unwrap_or(U256::ZERO); + + self.write_slot_through(address, slot, value); + + // Record only an actual change. + (old != value).then_some(SlotChange { + address, + slot, + old, + new: value, + }) + } + + /// The single dual-layer slot write path (§5.1), shared by [`apply_slot`], + /// the [`StateUpdate::SlotDelta`] handler, and [`modify_slot`](Self::modify_slot). + /// + /// Backend (layer 2) is always written; the overlay (layer 1) is written only + /// if an overlay account already exists. A new overlay account is never + /// materialized: that preserves the layer-2-only invariant (a fresh + /// `StorageCleared` overlay account would read missing slots as ZERO and could + /// shadow later RPC reads), and an absent overlay entry falls through to the + /// backend on reads so the backend write is authoritative. + fn write_slot_through(&mut self, address: Address, slot: U256, value: U256) { + // Backend (layer 2): always write. { let mut storage = self.blockchain_db.storage().write(); - for &(addr, slot, value) in results { - storage.entry(addr).or_default().insert(slot, value); - } + storage.entry(address).or_default().insert(slot, value); } - // Write through to the overlay only for accounts already materialized - // there, so the winning layer reflects the fresh value. - for &(addr, slot, value) in results { - if let Some(db_account) = self.db.cache.accounts.get_mut(&addr) { - db_account.storage.insert(slot, value); + + // Overlay (layer 1): write only if an overlay account already exists. + if let Some(db_account) = self.db.cache.accounts.get_mut(&address) { + db_account.storage.insert(slot, value); + } + } + + /// Read-modify-write one storage slot through a caller-supplied transform. + /// + /// The general closure escape hatch behind [`StateUpdate::SlotDelta`] (the + /// data-level form flows through [`apply_update`](Self::apply_update); this is + /// for arbitrary transforms). `f` is called with the current cached value + /// (overlay ▸ backend ▸ `None` when the slot is cold) and decides the new + /// value: + /// + /// - `Some(new)` writes `new` through both layers (the same write path as + /// [`StateUpdate::Slot`]) and returns a [`SlotChange`] iff it changed + /// (`old.unwrap_or(ZERO) != new`); + /// - `None` writes nothing and returns `None`. + /// + /// The caller owns the cold/overflow policy. To skip cold slots (the + /// cold-aware read-modify-write rule), map through the `Option`: + /// `|cur| cur.map(|v| v.saturating_add(amount))` leaves a cold slot untouched. + /// To write an absolute value regardless, ignore the argument: `|_| Some(v)`. + /// + /// ```no_run + /// # use alloy_primitives::{Address, U256}; + /// # fn example(cache: &mut evm_fork_cache::cache::EvmCache) { + /// let token = Address::repeat_byte(0x01); + /// let slot = U256::from(0); + /// // Saturating +100, but only if the slot is already hot. + /// let change = cache.modify_slot(token, slot, |cur| cur.map(|v| v.saturating_add(U256::from(100)))); + /// # let _ = change; + /// # } + /// ``` + pub fn modify_slot( + &mut self, + address: Address, + slot: U256, + f: impl FnOnce(Option) -> Option, + ) -> Option { + let current = self.cached_storage_value(address, slot); + let new = f(current)?; + + self.write_slot_through(address, slot, new); + + let old = current.unwrap_or(U256::ZERO); + (old != new).then_some(SlotChange { + address, + slot, + old, + new, + }) + } + + /// Read-modify-write an account's native balance through a caller-supplied + /// transform. + /// + /// The closure analog of [`StateUpdate::BalanceDelta`] (the data-level form + /// flows through [`apply_update`](Self::apply_update); this is for arbitrary + /// transforms). `f` is called with the account's current native balance + /// (overlay ▸ backend ▸ `None` when the account is absent from **both** + /// layers) and decides the new balance: + /// + /// - `Some(new)` writes `new` through both layers — backend always, overlay + /// only if an overlay account already exists — preserving the account's + /// nonce and code, and returns an [`AccountChange`] (balance only) iff the + /// balance changed; + /// - `None` writes nothing (no account is materialized) and returns `None`. + /// + /// "Cold" for a balance is the account being absent from both layers; the + /// revm `account_state` does **not** matter here (it governs storage, not the + /// basic `AccountInfo`). To skip cold accounts, map through the `Option`: + /// `|cur| cur.map(|v| v.saturating_add(amount))`. + /// + /// ```no_run + /// # use alloy_primitives::{Address, U256}; + /// # fn example(cache: &mut evm_fork_cache::cache::EvmCache) { + /// let acct = Address::repeat_byte(0x01); + /// // Saturating +100, but only if the account's balance is already known. + /// let change = cache.modify_account_balance(acct, |cur| cur.map(|v| v.saturating_add(U256::from(100)))); + /// # let _ = change; + /// # } + /// ``` + pub fn modify_account_balance( + &mut self, + address: Address, + f: impl FnOnce(Option) -> Option, + ) -> Option { + // Load the full info from the cached layers only (overlay ▸ backend); the + // account is "cold" when absent from both. + let base = self.loaded_account_info(address); + let current_balance = base.as_ref().map(|info| info.balance); + let new_balance = f(current_balance)?; + + // The closure asked to write `new_balance`. Materialize from the loaded + // base (or a default if the caller chose to write a cold account). + let mut info = base.unwrap_or_default(); + let old_balance = info.balance; + info.balance = new_balance; + self.write_account_info_through(address, info); + + (old_balance != new_balance).then_some(AccountChange { + address, + balance: Some((old_balance, new_balance)), + nonce: None, + code_hash: None, + }) + } + + /// Apply a relative (saturating) [`SlotDelta`] to an account's native balance + /// (§16.5). Cold-aware: + /// + /// - `Ok(Some(change))` — present account, balance changed; + /// - `Ok(None)` — present account, balance unchanged (e.g. `Sub` from 0); + /// - `Err(skipped)` — cold account (absent from both layers): nothing applied, + /// nothing materialized. + fn apply_balance_delta( + &mut self, + address: Address, + delta: SlotDelta, + ) -> std::result::Result, SkippedBalanceDelta> { + let Some(mut info) = self.loaded_account_info(address) else { + // Cold: applying a delta against an unknown balance would corrupt it, + // and materializing a default account would mask the real on-chain one. + return Err(SkippedBalanceDelta { address, delta }); + }; + + let old_balance = info.balance; + let new_balance = delta.apply(old_balance); + info.balance = new_balance; + self.write_account_info_through(address, info); + + Ok((old_balance != new_balance).then_some(AccountChange { + address, + balance: Some((old_balance, new_balance)), + nonce: None, + code_hash: None, + })) + } + + /// Load an account's `AccountInfo` from the cached layers only (overlay ▸ + /// backend), without touching RPC. `None` when the account is absent from + /// both layers. + fn loaded_account_info(&self, address: Address) -> Option { + self.db + .cache + .accounts + .get(&address) + .map(|a| a.info.clone()) + .or_else(|| self.blockchain_db.accounts().read().get(&address).cloned()) + } + + /// Write an `AccountInfo` through both layers, mirroring the slot policy: + /// backend (layer 2) always; overlay (layer 1) only if an overlay account + /// already exists (never materialize a new overlay account). + fn write_account_info_through(&mut self, address: Address, info: AccountInfo) { + let overlay_present = self.db.cache.accounts.contains_key(&address); + { + let mut accounts = self.blockchain_db.accounts().write(); + accounts.insert(address, info.clone()); + } + if overlay_present { + self.db.insert_account_info(address, info); + } + } + + /// Apply a partial [`AccountPatch`] write-through (§5.2). Returns an + /// [`AccountChange`] iff any field actually changes. + fn apply_account_patch( + &mut self, + address: Address, + patch: &AccountPatch, + ) -> Option { + // 1. Current info from the cached layers only (overlay ▸ backend ▸ + // default). No RPC: apply is a write, not a fetch. + let mut info = self.loaded_account_info(address).unwrap_or_default(); + + let old_balance = info.balance; + let old_nonce = info.nonce; + let old_code_hash = info.code_hash; + + // 2. Apply each `Some` field. + if let Some(balance) = patch.balance { + info.balance = balance; + } + if let Some(nonce) = patch.nonce { + info.nonce = nonce; + } + if let Some(code) = &patch.code { + let bytecode = Bytecode::new_raw(code.clone()); + info.code_hash = bytecode.hash_slow(); + info.code = Some(bytecode); + } + + // 3. Compute the change first. A no-op patch (every field equals the + // loaded base) must NOT write either layer — otherwise an all-`None` + // patch on an absent address would insert `AccountInfo::default()` into + // the shared backend (masking a future RPC fetch) while returning an + // empty diff. Only a real field change materializes anything. + let change = AccountChange { + address, + balance: (old_balance != info.balance).then_some((old_balance, info.balance)), + nonce: (old_nonce != info.nonce).then_some((old_nonce, info.nonce)), + code_hash: (old_code_hash != info.code_hash).then_some((old_code_hash, info.code_hash)), + }; + if change.balance.is_none() && change.nonce.is_none() && change.code_hash.is_none() { + return None; + } + + // 4. Write-through, mirroring the slot policy: backend always; overlay + // only if an overlay account already exists (do not materialize one). + self.write_account_info_through(address, info); + + Some(change) + } + + /// Dispatch a [`PurgeScope`] to the matching layer logic (§5.3), returning a + /// [`PurgeRecord`] of what was removed from each layer. + fn apply_purge(&mut self, address: Address, scope: &PurgeScope) -> PurgeRecord { + match scope { + PurgeScope::Account => { + let (slots_removed, account_removed) = self.purge_account_inner(address); + PurgeRecord { + address, + scope: PurgeScope::Account, + slots_removed, + account_removed, + } + } + PurgeScope::AllStorage => { + let slots_removed = self.purge_pool_storage_inner(address); + PurgeRecord { + address, + scope: PurgeScope::AllStorage, + slots_removed, + account_removed: false, + } + } + PurgeScope::Slots(slots) => { + let slots_removed = self.purge_pool_slots_inner(address, slots); + PurgeRecord { + address, + scope: PurgeScope::Slots(slots.clone()), + slots_removed, + account_removed: false, + } } } } @@ -1126,14 +1702,33 @@ impl EvmCache { /// Return the currently-cached value for a storage slot, if any. /// - /// Checks the CacheDB overlay (layer 1) first, then the BlockchainDb backend - /// (layer 2). Returns `None` when neither layer has seen the slot. Unlike - /// [`read_storage_slot`](Self::read_storage_slot) this never touches RPC. + /// Mirrors what the EVM would `SLOAD` from the cached layers (it never touches + /// RPC, unlike [`read_storage_slot`](Self::read_storage_slot)): + /// + /// 1. The CacheDB overlay (layer 1) wins: if the overlay account holds the + /// slot, return it. + /// 2. Match revm's `CacheDB::storage_ref`: if the overlay account exists but + /// does **not** hold the slot, and its `account_state` is `StorageCleared` + /// or `NotExisting`, the live EVM reads the slot as ZERO and never consults + /// the backend — so return `Some(U256::ZERO)`, **not** the (shadowed) + /// backend value. Returning the backend value here would let a + /// `SlotDelta`/`modify_slot` compute a delta against a base the EVM never + /// sees (silent corruption) and would mis-record `apply_slot`'s `old`. + /// 3. Otherwise fall through to the BlockchainDb backend (layer 2); `None` when + /// neither layer has seen the slot. pub fn cached_storage_value(&self, address: Address, slot: U256) -> Option { - if let Some(db_account) = self.db.cache.accounts.get(&address) - && let Some(value) = db_account.storage.get(&slot) - { - return Some(*value); + if let Some(db_account) = self.db.cache.accounts.get(&address) { + if let Some(value) = db_account.storage.get(&slot) { + return Some(*value); + } + // A StorageCleared / NotExisting overlay account reads a missing slot + // as ZERO and never consults the backend (matching the EVM SLOAD). + if matches!( + db_account.account_state, + AccountState::StorageCleared | AccountState::NotExisting + ) { + return Some(U256::ZERO); + } } let storage = self.blockchain_db.storage().read(); storage.get(&address).and_then(|s| s.get(&slot).copied()) @@ -1215,6 +1810,16 @@ impl EvmCache { /// use it when an address is fully volatile (no pinned slots) and even its /// balance/nonce/code can no longer be trusted. pub fn purge_account(&mut self, addr: Address) { + // Thin wrapper over the unified purge primitive; the layer logic lives in + // `purge_account_inner` (shared with `apply_update(Purge { Account })`). + let _ = self.apply_update(&StateUpdate::purge(addr, PurgeScope::Account)); + } + + /// Account-scope purge layer logic. Removes `addr` from the overlay accounts + /// map, the backend accounts map, and the backend storage map. Returns + /// `(backend_slots_removed, account_removed)` where `account_removed` is true + /// if an account entry was removed from either account layer. + fn purge_account_inner(&mut self, addr: Address) -> (usize, bool) { // Layer 1: CacheDB overlay (accounts + their storage live together). let overlay_removed = self.db.cache.accounts.remove(&addr).is_some(); @@ -1225,17 +1830,22 @@ impl EvmCache { .write() .remove(&addr) .is_some(); - let backend_storage_removed = self.blockchain_db.storage().write().remove(&addr).is_some(); + let backend_storage_removed = self.blockchain_db.storage().write().remove(&addr); + let slots_removed = backend_storage_removed + .map(|slots| slots.len()) + .unwrap_or(0); - if overlay_removed || backend_account_removed || backend_storage_removed { + let account_removed = overlay_removed || backend_account_removed; + if account_removed || slots_removed > 0 { debug!( account = %addr, overlay_removed, backend_account_removed, - backend_storage_removed, + backend_storage_slots = slots_removed, "purged account from both cache layers" ); } + (slots_removed, account_removed) } /// Get the chain ID used for EVM simulations (the `CHAINID` opcode). @@ -1669,6 +2279,13 @@ impl EvmCache { /// # Arguments /// * `pool_address` - The UniswapV2 pair contract address /// * `metadata` - The cached pool metadata containing token0 and token1 + /// + /// # Layering (Phase 3 change) + /// As of Phase 3 this writes **through** the dual-layer policy via + /// [`apply_updates`](Self::apply_updates) (backend always, overlay-if-present) + /// rather than the old overlay-only write. The slot *placement* is normalized; + /// the visible `token0()` / `token1()` reads are unchanged. The slot writes are + /// now infallible; the `Result` is retained for signature compatibility. #[cfg(feature = "protocols")] #[cfg_attr(docsrs, doc(cfg(feature = "protocols")))] pub fn inject_v2_pool_metadata( @@ -1683,10 +2300,10 @@ impl EvmCache { let token0_value = U256::from_be_slice(metadata.token0.as_slice()); let token1_value = U256::from_be_slice(metadata.token1.as_slice()); - self.db - .insert_account_storage(pool_address, TOKEN0_SLOT, token0_value)?; - self.db - .insert_account_storage(pool_address, TOKEN1_SLOT, token1_value)?; + self.apply_updates(&[ + StateUpdate::slot(pool_address, TOKEN0_SLOT, token0_value), + StateUpdate::slot(pool_address, TOKEN1_SLOT, token1_value), + ]); Ok(()) } @@ -1703,6 +2320,14 @@ impl EvmCache { /// # Arguments /// * `pool_address` - The UniswapV3 pool contract address /// * `tick_bitmap` - Map of word position (int16) to bitmap value (uint256) + /// + /// # Layering (Phase 3 change) + /// As of Phase 3 this writes **through** the dual-layer policy via + /// [`apply_updates`](Self::apply_updates) (backend always, overlay-if-present) + /// rather than the old overlay-only write — so the slots now land in the + /// BlockchainDb backend (layer 2) too. See `CHANGELOG.md` / `KNOWN_ISSUES.md`. + /// The slot writes are now infallible; the `Result` is retained for signature + /// compatibility. #[cfg(feature = "protocols")] #[cfg_attr(docsrs, doc(cfg(feature = "protocols")))] pub fn inject_v3_tick_bitmap( @@ -1724,17 +2349,17 @@ impl EvmCache { tick_bitmap: &std::collections::HashMap, base_slot: U256, ) -> Result { - let mut injected = 0; + let mut updates = Vec::with_capacity(tick_bitmap.len()); for (&word_position, &bitmap_value) in tick_bitmap { let word_position_i256 = i256_from_i16(word_position); let mut slot_preimage = [0u8; 64]; slot_preimage[..32].copy_from_slice(&word_position_i256); slot_preimage[32..64].copy_from_slice(&base_slot.to_be_bytes::<32>()); let storage_slot: U256 = keccak256(slot_preimage).into(); - self.db - .insert_account_storage(pool_address, storage_slot, bitmap_value)?; - injected += 1; + updates.push(StateUpdate::slot(pool_address, storage_slot, bitmap_value)); } + let injected = updates.len(); + self.apply_updates(&updates); Ok(injected) } @@ -1762,6 +2387,13 @@ impl EvmCache { /// # Arguments /// * `pool_address` - The UniswapV3 pool contract address /// * `ticks` - Map of tick index (int24) to tick info + /// + /// # Layering (Phase 3 change) + /// As of Phase 3 this writes **through** the dual-layer policy via + /// [`apply_updates`](Self::apply_updates) (backend always, overlay-if-present) + /// rather than the old overlay-only write. See `CHANGELOG.md` / + /// `KNOWN_ISSUES.md`. The slot writes are now infallible; the `Result` is + /// retained for signature compatibility. #[cfg(feature = "protocols")] #[cfg_attr(docsrs, doc(cfg(feature = "protocols")))] pub fn inject_v3_ticks( @@ -1783,7 +2415,7 @@ impl EvmCache { ticks: &std::collections::HashMap, ticks_slot: U256, ) -> Result { - let mut injected = 0; + let mut updates = Vec::with_capacity(ticks.len() * 2); for (&tick, info) in ticks { let tick_i256 = i256_from_i24(tick); let mut slot_preimage = [0u8; 64]; @@ -1798,8 +2430,7 @@ impl EvmCache { let liquidity_net_u256 = i128_to_u256(info.liquidity_net); let packed_slot0 = liquidity_gross_u256 | (liquidity_net_u256 << 128); - self.db - .insert_account_storage(pool_address, base_slot, packed_slot0)?; + updates.push(StateUpdate::slot(pool_address, base_slot, packed_slot0)); // Also inject slot 3 with the `initialized` flag. // Slot 3 layout: packed (tickCumulativeOutside, secondsPerLiquidityOutsideX128, @@ -1818,12 +2449,11 @@ impl EvmCache { } else { U256::ZERO }; - self.db - .insert_account_storage(pool_address, slot3, initialized_value)?; - - injected += 1; + updates.push(StateUpdate::slot(pool_address, slot3, initialized_value)); } + let injected = ticks.len(); + self.apply_updates(&updates); Ok(injected) } @@ -2167,6 +2797,19 @@ impl EvmCache { /// After purging both layers, the next EVM read for this pool's storage will /// go all the way to the RPC for fresh data. pub fn purge_pool_storage(&mut self, address: Address) -> usize { + // Thin wrapper over the unified purge primitive; returns the backend slot + // count the `AllStorage` scope removed. + self.apply_update(&StateUpdate::purge(address, PurgeScope::AllStorage)) + .purged + .first() + .map(|rec| rec.slots_removed) + .unwrap_or(0) + } + + /// `AllStorage`-scope purge layer logic. Clears the overlay storage for + /// `address` and removes its backend storage map. Returns the number of + /// backend slots removed. + fn purge_pool_storage_inner(&mut self, address: Address) -> usize { // Layer 1: Clear CacheDB overlay let cache_db_cleared = if let Some(db_account) = self.db.cache.accounts.get_mut(&address) { let count = db_account.storage.len(); @@ -2206,6 +2849,21 @@ impl EvmCache { /// /// Returns the number of slots removed from the BlockchainDb backend. pub fn purge_pool_slots(&mut self, address: Address, slots: &[U256]) -> usize { + // Thin wrapper over the unified purge primitive; returns the backend slot + // count the `Slots` scope removed. + self.apply_update(&StateUpdate::purge( + address, + PurgeScope::Slots(slots.to_vec()), + )) + .purged + .first() + .map(|rec| rec.slots_removed) + .unwrap_or(0) + } + + /// `Slots`-scope purge layer logic. Removes the listed slots from the overlay + /// and the backend storage map. Returns the number of backend slots removed. + fn purge_pool_slots_inner(&mut self, address: Address, slots: &[U256]) -> usize { let mut cache_db_removed = 0usize; let mut backend_removed = 0usize; @@ -2644,6 +3302,17 @@ impl EvmCache { } /// Override code at `target`, with explicit behavior for missing target accounts. + /// + /// This is intentionally **not** folded onto + /// [`apply_update`](Self::apply_update)'s `Account` code patch: it copies code + /// from a `source` account, preserves the target's existing balance/nonce/ + /// storage, and **unconditionally materializes** the target in the CacheDB + /// overlay (the primary read path for EVM execution, required for the + /// `Create` synthetic-target case). The generic primitive writes the overlay + /// only when an account is already present, so the two are not + /// behavior-equivalent. For a plain code overwrite that follows the + /// dual-layer write-through policy, use + /// `apply_update(StateUpdate::Account { patch: AccountPatch::default().code(..) })`. pub fn override_account_code_with_missing_target( &mut self, source: Address, diff --git a/src/freshness.rs b/src/freshness.rs index 302c301..0fce064 100644 --- a/src/freshness.rs +++ b/src/freshness.rs @@ -66,6 +66,7 @@ use crate::cache::{ CallSimulationResult, EvmCache, EvmOverlay, EvmSnapshot, SimStatus, SlotObservationTracker, StorageBatchFetchFn, TxConfig, }; +use crate::state_update::StateUpdate; /// Default minimum observations before the change-frequency data is trusted. pub const DEFAULT_MIN_OBSERVATIONS: u32 = 10; @@ -434,12 +435,17 @@ impl FreshnessPolicy for ObservationDriven { // 4. Results // --------------------------------------------------------------------------- -/// A storage slot whose freshly-fetched value differs from the cached value. +/// A storage slot whose value changed: `old` is the prior cached/snapshot value +/// (`ZERO` if previously uncached), `new` is the resulting value. /// -/// Produced by [`EvmCache::verify_slots`](crate::cache::EvmCache::verify_slots) -/// and by the background validator; `old` is the value the snapshot/cache held, -/// `new` is the value the fetcher returned. -#[derive(Clone, Debug, PartialEq, Eq)] +/// Produced by two paths: the freshness verifier +/// ([`EvmCache::verify_slots`](crate::cache::EvmCache::verify_slots) and the +/// background validator), where `new` is a freshly-fetched value that differed +/// from the cache; and the state-update writer +/// ([`EvmCache::apply_update`](crate::cache::EvmCache::apply_update) / +/// [`apply_updates`](crate::cache::EvmCache::apply_updates)), where `new` is the +/// value just written. +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct SlotChange { /// Contract whose storage changed. pub address: Address, @@ -777,12 +783,17 @@ impl FreshnessController { let now = self.clock.now(); // 1. Drain pending corrections into the cache before snapshotting. + // Routed through the unified write primitive (`apply_updates` of + // write-through `Slot`s); behavior-identical to the old + // `inject_storage_batch_fresh`, demonstrating the one write path. { let mut pending = self.pending.lock().unwrap_or_else(|e| e.into_inner()); if !pending.is_empty() { - let injects: Vec<(Address, U256, U256)> = - pending.iter().map(|c| (c.address, c.slot, c.new)).collect(); - cache.inject_storage_batch_fresh(&injects); + let injects: Vec = pending + .iter() + .map(|c| StateUpdate::slot(c.address, c.slot, c.new)) + .collect(); + cache.apply_updates(&injects); pending.clear(); } } diff --git a/src/lib.rs b/src/lib.rs index 434c451..7428463 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -45,6 +45,10 @@ //! - [`freshness`] — the four-layer freshness model (classification, observation, //! policy, mechanism) and the optimistic verify-and-rerun execution loop with //! deferred validation. +//! - [`state_update`] — the generic state-mutation vocabulary (`StateUpdate` / +//! `AccountPatch` / `PurgeScope`, plus relative `SlotDelta` read-modify-write) +//! applied by `EvmCache::apply_update` / `apply_updates` / `modify_slot`, with a +//! structured `StateDiff` output (Pillar B.1). //! - [`inspector`] — an [`Inspector`](revm::Inspector) that captures ERC20 //! `Transfer` events to reconstruct balance deltas from a simulation. //! - [`multicall`] — batched read-only calls through Multicall3. @@ -102,6 +106,7 @@ pub mod freshness; pub mod inspector; pub mod multicall; pub mod prefetch_registry; +pub mod state_update; pub use access_set::StorageAccessList; pub use freshness::{ @@ -109,3 +114,7 @@ pub use freshness::{ FreshnessPolicy, FreshnessRegistry, NeverVerify, ObservationDriven, SimRequest, SlotChange, SpeculativeSim, Validation, Validity, WallClock, }; +pub use state_update::{ + AccountChange, AccountPatch, PurgeRecord, PurgeScope, SkippedBalanceDelta, SkippedDelta, + SlotDelta, StateDiff, StateUpdate, +}; diff --git a/src/state_update.rs b/src/state_update.rs new file mode 100644 index 0000000..8fc18b7 --- /dev/null +++ b/src/state_update.rs @@ -0,0 +1,696 @@ +//! Targeted state-mutation vocabulary and the structured diff it produces +//! (Pillar B.1 — the *writer half* of the event → state pipeline). +//! +//! This module defines the small, generic vocabulary a future event decoder +//! emits and [`EvmCache::apply_update`](crate::cache::EvmCache::apply_update) +//! consumes, plus the [`StateDiff`] that records what an apply actually changed. +//! It is pure data and logic on itself: it carries **no** protocol or event +//! knowledge and has no dependency on the cache or the `protocols` feature, so +//! it builds under `--no-default-features`. +//! +//! # The vocabulary +//! +//! A [`StateUpdate`] is one targeted mutation: +//! +//! - [`StateUpdate::Slot`] — set a single storage slot, authoritative across +//! both cache layers. +//! - [`StateUpdate::Account`] — apply a partial [`AccountPatch`] +//! (`balance`/`nonce`/`code`, each optional). +//! - [`StateUpdate::Purge`] — drop cached state at a [`PurgeScope`] so the next +//! read re-fetches. +//! +//! # The dual-layer write-through policy +//! +//! [`apply_update`](crate::cache::EvmCache::apply_update) applies a `Slot` or +//! `Account` write-through with one consistent rule: the BlockchainDb backend +//! (layer 2) is written **always**; the CacheDB overlay (layer 1) is written +//! **only if an overlay account already exists** for the address. A new overlay +//! account is never materialized for a slot/account write — the read path falls +//! through to the backend for an absent overlay entry, so a backend-only write +//! is authoritative, and materializing an overlay entry would pollute layer 1 +//! and could shadow later RPC reads. (This mirrors the established +//! [`inject_storage_batch_fresh`](crate::cache::EvmCache::inject_storage_batch_fresh) +//! semantics.) +//! +//! # The output +//! +//! Every apply returns a [`StateDiff`] of the changes it actually made: the +//! [`SlotChange`]s, [`AccountChange`]s, and [`PurgeRecord`]s. **Only real changes +//! are recorded** — re-applying a value the cache already holds yields an empty +//! diff, so idempotence is observable. +//! +//! # Relative updates / cold-aware read-modify-write +//! +//! Some callers learn only a *delta* (an ERC-20 `Transfer` log carries the +//! transferred `amount`, not the resulting balances), so the vocabulary also +//! supports *relative* updates: [`StateUpdate::SlotDelta`] reads the current slot +//! value, applies a saturating [`SlotDelta`] (`Add` clamps at `U256::MAX`, `Sub` +//! at `U256::ZERO`), and writes the result back through both layers. The general +//! closure form is +//! [`EvmCache::modify_slot`](crate::cache::EvmCache::modify_slot). +//! +//! A relative update is only valid against a value the cache *actually holds*. An +//! un-fetched ("cold") slot has no value, and applying a delta to it would compute +//! `0 ± amount`, write a wrong value, and (write-through) make it authoritative — +//! silently corrupting state. So relative application is **cold-aware**: a +//! `SlotDelta` on a cold slot is **not applied**; it is recorded in +//! [`StateDiff::skipped`] as a [`SkippedDelta`] so the caller can fetch+seed the +//! true value (the next read otherwise lazily fetches it). `modify_slot` hands its +//! closure an `Option` (`None` when cold) and lets the caller decide. +//! +//! The same relative, cold-aware rule extends to an account's **native balance**: +//! [`StateUpdate::BalanceDelta`] (and the closure form +//! [`EvmCache::modify_account_balance`](crate::cache::EvmCache::modify_account_balance)) +//! read-modify-write `AccountInfo::balance`, preserving nonce/code. "Cold" here +//! means the account is absent from *both* layers (its balance is unknown); a +//! `BalanceDelta` on a cold account is **not applied** — it is surfaced in +//! [`StateDiff::skipped_balances`] as a [`SkippedBalanceDelta`]. This avoids +//! materializing a default account that would mask the real on-chain one. +//! +//! # Checking for skips +//! +//! Because a cold-skipped relative update produces **no** change, it is invisible +//! to the natural [`StateDiff::is_empty`] / [`StateDiff::len`] success check (those +//! are changes-only). A caller applying relative updates **must** therefore check +//! [`StateDiff::has_skipped`] (or inspect [`skipped`](StateDiff::skipped) / +//! [`skipped_balances`](StateDiff::skipped_balances)) — a cold target was dropped, +//! not applied, and a silently-dropped balance update can break conservation. +//! [`StateDiff::is_fully_applied`] and [`StateDiff::skipped_len`] are the +//! companions. +//! +//! # Warning — cold absolute `Account` patches +//! +//! A *partial* absolute [`StateUpdate::Account`] patch (e.g. balance-only) on an +//! address absent from **both** cache layers writes default nonce/code through the +//! shared backend as authoritative, pre-empting a real RPC fetch. Fetch+seed the +//! account first, or prefer [`StateUpdate::BalanceDelta`] for relative +//! native-balance tracking. See the warnings on +//! [`apply_update`](crate::cache::EvmCache::apply_update), +//! [`StateUpdate::Account`], and [`AccountPatch`]. +//! +//! # Boundary — events are Phase 4 +//! +//! This is the vocabulary a Phase 4 `EventDecoder` will *emit into*; Phase 3 +//! does not decode events. Nothing here parses a `Log` or knows a protocol's +//! storage layout — that is the *reader half* of Pillar B and lands later. + +use alloy_primitives::{Address, B256, Bytes, U256}; + +use crate::freshness::SlotChange; + +/// A relative storage-slot mutation: read the current value, transform it, and +/// write it back. +/// +/// Both directions **saturate** rather than wrap: `Add` clamps at `U256::MAX` +/// and `Sub` clamps at `U256::ZERO`. This is the delta a caller derives from an +/// event (e.g. an ERC-20 `Transfer` amount) without knowing the resulting +/// absolute value. It is applied by [`StateUpdate::SlotDelta`] (cold-aware — see +/// the module docs). +/// +/// ``` +/// use alloy_primitives::U256; +/// use evm_fork_cache::SlotDelta; +/// +/// assert_eq!(SlotDelta::Add(U256::from(50)).apply(U256::from(100)), U256::from(150)); +/// assert_eq!(SlotDelta::Sub(U256::from(50)).apply(U256::from(30)), U256::ZERO); +/// assert_eq!(SlotDelta::Add(U256::from(10)).apply(U256::MAX), U256::MAX); +/// ``` +#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub enum SlotDelta { + /// Add to the current value, saturating at `U256::MAX`. + Add(U256), + /// Subtract from the current value, saturating at `U256::ZERO`. + Sub(U256), +} + +impl SlotDelta { + /// Apply the (saturating) delta to a current value. + /// + /// `Add` uses `saturating_add` (clamps at `U256::MAX`); `Sub` uses + /// `saturating_sub` (clamps at `U256::ZERO`). + pub fn apply(self, current: U256) -> U256 { + match self { + SlotDelta::Add(amount) => current.saturating_add(amount), + SlotDelta::Sub(amount) => current.saturating_sub(amount), + } + } +} + +/// A single targeted mutation to cached EVM state. +/// +/// The vocabulary an event decoder (Phase 4) emits and +/// [`EvmCache::apply_update`](crate::cache::EvmCache::apply_update) consumes. +/// Generic: carries no protocol or event knowledge. +/// +/// The enum is `#[non_exhaustive]`: new variants (e.g. a code-only convenience) +/// may be added pre-1.0 without a breaking change. +/// +/// ``` +/// use alloy_primitives::{Address, U256}; +/// use evm_fork_cache::{AccountPatch, PurgeScope, StateUpdate}; +/// +/// let pool = Address::repeat_byte(0x01); +/// +/// // A storage-slot write (authoritative across both cache layers). +/// let slot = StateUpdate::slot(pool, U256::from(0), U256::from(42)); +/// +/// // A balance-only account patch (nonce and code left untouched). +/// let bal = StateUpdate::balance(pool, U256::from(1_000)); +/// assert_eq!( +/// bal, +/// StateUpdate::Account { address: pool, patch: AccountPatch::default().balance(U256::from(1_000)) }, +/// ); +/// +/// // Drop just two storage slots so the next read re-fetches them. +/// let purge = StateUpdate::purge(pool, PurgeScope::Slots(vec![U256::from(0), U256::from(1)])); +/// # let _ = (slot, purge); +/// ``` +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[non_exhaustive] +pub enum StateUpdate { + /// Set one storage slot to `value`, authoritative across both cache layers. + Slot { + /// Contract whose storage is written. + address: Address, + /// Storage slot key. + slot: U256, + /// New slot value. + value: U256, + }, + /// Apply a *relative* (saturating) mutation to one storage slot. + /// + /// Read-modify-write: the current value is read, the [`SlotDelta`] applied, + /// and the result written back through both layers. **Cold-aware** — a delta + /// on a slot absent from both layers is not applied; it is surfaced in + /// [`StateDiff::skipped`] instead (see the module docs). + SlotDelta { + /// Contract whose storage is written. + address: Address, + /// Storage slot key. + slot: U256, + /// The relative, saturating mutation to apply to the current value. + delta: SlotDelta, + }, + /// Apply a *relative* (saturating) mutation to an account's **native balance**. + /// + /// Read-modify-write: the current `AccountInfo::balance` is read, the + /// [`SlotDelta`] applied, and the result written back through both layers + /// (nonce and code preserved). **Cold-aware** — "cold" here means the account + /// is absent from *both* layers (its balance is unknown). A `BalanceDelta` on a + /// cold account is not applied; it is surfaced in + /// [`StateDiff::skipped_balances`] instead (so no default account is + /// materialized to mask the real on-chain one — see the module docs). + BalanceDelta { + /// Account whose native balance is mutated. + address: Address, + /// The relative, saturating mutation to apply to the current balance. + delta: SlotDelta, + }, + /// Patch an account's balance/nonce/code (partial — see [`AccountPatch`]). + /// + /// # Warning + /// + /// A partial absolute patch (e.g. balance-only) on an address absent from + /// **both** cache layers writes default nonce/code through the shared backend + /// as authoritative, pre-empting a real RPC fetch. Fetch+seed the account + /// first, or use [`StateUpdate::BalanceDelta`] for relative native-balance + /// tracking. + Account { + /// Account to patch. + address: Address, + /// The partial mutation: each `Some` field overwrites, `None` leaves it. + patch: AccountPatch, + }, + /// Purge cached state for `address` at `scope`; the next read re-fetches. + Purge { + /// Account whose cached state is purged. + address: Address, + /// What part of the cached state to remove. + scope: PurgeScope, + }, +} + +impl StateUpdate { + /// Construct a [`StateUpdate::Slot`] that sets `(address, slot)` to `value`. + pub fn slot(address: Address, slot: U256, value: U256) -> Self { + Self::Slot { + address, + slot, + value, + } + } + + /// Construct a [`StateUpdate::SlotDelta`] that applies `delta` relative to the + /// current value of `(address, slot)`. + pub fn slot_delta(address: Address, slot: U256, delta: SlotDelta) -> Self { + Self::SlotDelta { + address, + slot, + delta, + } + } + + /// Construct a [`StateUpdate::BalanceDelta`] that applies `delta` relative to + /// the account's current native balance. + pub fn balance_delta(address: Address, delta: SlotDelta) -> Self { + Self::BalanceDelta { address, delta } + } + + /// Construct a [`StateUpdate::Account`] that patches only the balance. + pub fn balance(address: Address, value: U256) -> Self { + Self::Account { + address, + patch: AccountPatch::default().balance(value), + } + } + + /// Construct a [`StateUpdate::Account`] that patches only the nonce. + pub fn nonce(address: Address, nonce: u64) -> Self { + Self::Account { + address, + patch: AccountPatch::default().nonce(nonce), + } + } + + /// Construct a [`StateUpdate::Account`] that patches only the runtime code + /// (the code hash is recomputed from `code` when applied). + pub fn code(address: Address, code: Bytes) -> Self { + Self::Account { + address, + patch: AccountPatch::default().code(code), + } + } + + /// Construct a [`StateUpdate::Account`] from a prebuilt [`AccountPatch`]. + pub fn account(address: Address, patch: AccountPatch) -> Self { + Self::Account { address, patch } + } + + /// Construct a [`StateUpdate::Purge`] for `address` at `scope`. + pub fn purge(address: Address, scope: PurgeScope) -> Self { + Self::Purge { address, scope } + } +} + +/// A partial account mutation: each `Some` field overwrites the cached value, +/// each `None` leaves it unchanged. Setting `code` recomputes the code hash; +/// `Some(empty bytes)` clears code to the empty-code hash. +/// +/// Partial (rather than a full revm `AccountInfo`) because the Pillar B driver +/// is events, which usually carry *one* field (a `Transfer` changes a balance, +/// not nonce/code). This avoids forcing a caller to reconstruct a full +/// `AccountInfo` and keeps revm's type out of the public vocabulary. +/// +/// The struct is `#[non_exhaustive]`: new fields may be added pre-1.0 without a +/// breaking change. Construct it via [`AccountPatch::default`] + the builders +/// ([`balance`](Self::balance) / [`nonce`](Self::nonce) / [`code`](Self::code)), +/// never a struct literal. +/// +/// # Warning +/// +/// Applying an absolute patch with [`StateUpdate::Account`] on an address absent +/// from **both** cache layers writes default values for the un-patched fields +/// (e.g. nonce `0`, empty code) through the shared backend as authoritative, +/// masking a later RPC fetch of the real on-chain account. Fetch+seed the account +/// first, or use [`StateUpdate::BalanceDelta`] for relative native-balance +/// tracking. +/// +/// ``` +/// use alloy_primitives::{Bytes, U256}; +/// use evm_fork_cache::AccountPatch; +/// +/// let patch = AccountPatch::default() +/// .balance(U256::from(42)) +/// .nonce(7) +/// .code(Bytes::from_static(&[0x60, 0x00])); +/// assert_eq!(patch.balance, Some(U256::from(42))); +/// assert_eq!(patch.nonce, Some(7)); +/// assert_eq!(AccountPatch::default().balance, None); +/// ``` +#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[non_exhaustive] +pub struct AccountPatch { + /// New balance, if set. + pub balance: Option, + /// New nonce, if set. + pub nonce: Option, + /// New runtime code, if set. Setting it recomputes the code hash; empty + /// bytes clear the code to the empty-code hash. + pub code: Option, +} + +impl AccountPatch { + /// Set the balance to overwrite (builder style). + pub fn balance(mut self, balance: U256) -> Self { + self.balance = Some(balance); + self + } + + /// Set the nonce to overwrite (builder style). + pub fn nonce(mut self, nonce: u64) -> Self { + self.nonce = Some(nonce); + self + } + + /// Set the runtime code to overwrite (builder style). The code hash is + /// recomputed from these bytes when the patch is applied. + pub fn code(mut self, code: Bytes) -> Self { + self.code = Some(code); + self + } +} + +/// What part of an address's cached state a purge removes. +/// +/// The enum is `#[non_exhaustive]`: new scopes may be added pre-1.0 without a +/// breaking change. +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[non_exhaustive] +pub enum PurgeScope { + /// Full account: `AccountInfo` (balance/nonce/code) **and** all storage. + /// Equivalent to + /// [`EvmCache::purge_account`](crate::cache::EvmCache::purge_account). + Account, + /// All storage slots; account info preserved. Equivalent to + /// [`EvmCache::purge_pool_storage`](crate::cache::EvmCache::purge_pool_storage). + AllStorage, + /// Only the listed storage slots. Equivalent to + /// [`EvmCache::purge_pool_slots`](crate::cache::EvmCache::purge_pool_slots). + Slots(Vec), +} + +/// What an `apply_*` call actually changed. +/// +/// Returned by [`EvmCache::apply_update`](crate::cache::EvmCache::apply_update) +/// and [`apply_updates`](crate::cache::EvmCache::apply_updates). Only real +/// changes are recorded, so a no-op write yields a [`Default`] (empty) diff. +/// +/// The struct is `#[non_exhaustive]`: it has grown fields pre-1.0 +/// ([`skipped`](Self::skipped), [`skipped_balances`](Self::skipped_balances)) and +/// may grow more. Construct it via [`Default`] + field assignment, never an +/// exhaustive struct literal. +/// +/// # Checking for skips +/// +/// [`is_empty`](Self::is_empty) / [`len`](Self::len) are **changes-only**, so a +/// cold-skipped relative update ([`SlotDelta`](StateUpdate::SlotDelta) / +/// [`BalanceDelta`](StateUpdate::BalanceDelta)) is invisible to them. After +/// applying relative updates, check [`has_skipped`](Self::has_skipped) (or +/// inspect [`skipped`](Self::skipped) / [`skipped_balances`](Self::skipped_balances)) +/// — a cold target was dropped, not applied. +#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[non_exhaustive] +pub struct StateDiff { + /// Storage slots whose value changed (`old != new`). + pub slots: Vec, + /// Accounts whose balance/nonce/code-hash changed. + pub accounts: Vec, + /// Purges performed, with what they removed. + pub purged: Vec, + /// Relative slot updates ([`StateUpdate::SlotDelta`]) that were **not** applied + /// because the target slot's current value was unknown (cold). This is + /// informational metadata, not a change: it does **not** affect + /// [`is_empty`](Self::is_empty) / [`len`](Self::len). + pub skipped: Vec, + /// Relative balance updates ([`StateUpdate::BalanceDelta`]) that were **not** + /// applied because the target account was absent from both layers (its balance + /// was unknown). Like [`skipped`](Self::skipped) this is informational + /// metadata, not a change. + pub skipped_balances: Vec, +} + +impl StateDiff { + /// Whether the diff recorded no change at all. + /// + /// Changes-only: counts `slots` + `accounts` + `purged`. A skipped relative + /// update ([`skipped`](Self::skipped) / [`skipped_balances`](Self::skipped_balances)) + /// is informational metadata, not a change, so it does not affect this. + pub fn is_empty(&self) -> bool { + self.slots.is_empty() && self.accounts.is_empty() && self.purged.is_empty() + } + + /// Total number of changed entries (slots + accounts + purges). + /// + /// Changes-only: skipped relative updates are not counted (a skip is not a + /// change). See [`skipped_len`](Self::skipped_len) for the skip count. + pub fn len(&self) -> usize { + self.slots.len() + self.accounts.len() + self.purged.len() + } + + /// Whether any relative update was skipped (slot **or** balance). + /// + /// `true` iff [`skipped`](Self::skipped) or + /// [`skipped_balances`](Self::skipped_balances) is non-empty. A cold-skipped + /// update produces no change, so it is invisible to + /// [`is_empty`](Self::is_empty) — callers applying relative updates should + /// check this to avoid silently dropping a balance update. + pub fn has_skipped(&self) -> bool { + !self.skipped.is_empty() || !self.skipped_balances.is_empty() + } + + /// Total number of skipped relative updates (`skipped` + `skipped_balances`). + pub fn skipped_len(&self) -> usize { + self.skipped.len() + self.skipped_balances.len() + } + + /// Whether every relative update in the apply was applied (none skipped). + /// + /// The inverse of [`has_skipped`](Self::has_skipped). + pub fn is_fully_applied(&self) -> bool { + !self.has_skipped() + } + + /// Fold `other` into `self`, concatenating each category. + /// + /// Used by [`apply_updates`](crate::cache::EvmCache::apply_updates) to merge + /// per-update diffs; the concatenation preserves order, so two writes to the + /// same slot record their `old → new` history in sequence. The `skipped` and + /// `skipped_balances` metadata are concatenated too. + pub fn merge(&mut self, other: StateDiff) { + self.slots.extend(other.slots); + self.accounts.extend(other.accounts); + self.purged.extend(other.purged); + self.skipped.extend(other.skipped); + self.skipped_balances.extend(other.skipped_balances); + } +} + +/// An account field delta. Each field is `Some((old, new))` only when it changed. +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct AccountChange { + /// Account whose fields changed. + pub address: Address, + /// Balance delta `(old, new)`, present only if the balance changed. + pub balance: Option<(U256, U256)>, + /// Nonce delta `(old, new)`, present only if the nonce changed. + pub nonce: Option<(u64, u64)>, + /// Code-hash delta `(old, new)`, present only if the code changed. + pub code_hash: Option<(B256, B256)>, +} + +/// Record of a purge: how much of each layer it removed. +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct PurgeRecord { + /// Account that was purged. + pub address: Address, + /// The scope that was applied. + pub scope: PurgeScope, + /// Storage slots removed from the BlockchainDb backend (layer 2). + pub slots_removed: usize, + /// Whether an `AccountInfo` was removed (only the [`PurgeScope::Account`] scope). + pub account_removed: bool, +} + +/// A relative update ([`StateUpdate::SlotDelta`]) that could not be applied +/// because the slot's current value is unknown (not cached in either layer). +/// +/// A delta against a cold slot is skipped rather than applied (applying `0 ± +/// amount` would corrupt an unknown value and, write-through, make it +/// authoritative). It is surfaced here so the caller can fetch+seed the true +/// value and retry; otherwise the next read lazily fetches it. +#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct SkippedDelta { + /// Contract whose storage slot the delta targeted. + pub address: Address, + /// Storage slot key that was cold. + pub slot: U256, + /// The delta that was not applied. + pub delta: SlotDelta, +} + +/// A relative balance update ([`StateUpdate::BalanceDelta`]) that could not be +/// applied because the account is absent from **both** cache layers (its native +/// balance is unknown). +/// +/// A delta against a cold account is skipped rather than applied (applying it +/// against an assumed-zero balance would corrupt an unknown value, and +/// materializing a default account would mask the real on-chain one). It is +/// surfaced here so the caller can fetch+seed the account and retry. +/// +/// Deliberately **not** `#[non_exhaustive]`: it is a stable, fully-determined leaf +/// record routinely constructed as a struct literal in equality assertions by the +/// test suite and downstream users testing against a returned diff. +#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct SkippedBalanceDelta { + /// Account whose native balance the delta targeted. + pub address: Address, + /// The delta that was not applied. + pub delta: SlotDelta, +} + +#[cfg(test)] +mod tests { + use super::*; + + fn addr(n: u8) -> Address { + Address::repeat_byte(n) + } + + #[test] + fn account_patch_default_is_all_none() { + let p = AccountPatch::default(); + assert_eq!(p.balance, None); + assert_eq!(p.nonce, None); + assert_eq!(p.code, None); + } + + #[test] + fn account_patch_builders_compose() { + let p = AccountPatch::default() + .balance(U256::from(42)) + .nonce(7) + .code(Bytes::from_static(&[0x60, 0x00])); + assert_eq!(p.balance, Some(U256::from(42))); + assert_eq!(p.nonce, Some(7)); + assert_eq!(p.code, Some(Bytes::from_static(&[0x60, 0x00]))); + } + + #[test] + fn state_update_constructors_produce_expected_variants() { + let a = addr(0xaa); + + assert_eq!( + StateUpdate::slot(a, U256::from(1), U256::from(2)), + StateUpdate::Slot { + address: a, + slot: U256::from(1), + value: U256::from(2), + } + ); + assert_eq!( + StateUpdate::balance(a, U256::from(9)), + StateUpdate::Account { + address: a, + patch: AccountPatch::default().balance(U256::from(9)), + } + ); + assert_eq!( + StateUpdate::purge(a, PurgeScope::Account), + StateUpdate::Purge { + address: a, + scope: PurgeScope::Account, + } + ); + } + + #[test] + fn state_diff_default_is_empty() { + let d = StateDiff::default(); + assert!(d.is_empty()); + assert_eq!(d.len(), 0); + } + + #[test] + fn state_diff_merge_concatenates_and_counts() { + let a = addr(0xbb); + let mut left = StateDiff::default(); + left.slots.push(SlotChange { + address: a, + slot: U256::from(1), + old: U256::ZERO, + new: U256::from(5), + }); + + let mut right = StateDiff::default(); + right.accounts.push(AccountChange { + address: a, + balance: Some((U256::ZERO, U256::from(3))), + nonce: None, + code_hash: None, + }); + right.purged.push(PurgeRecord { + address: a, + scope: PurgeScope::AllStorage, + slots_removed: 2, + account_removed: false, + }); + + left.merge(right); + assert!(!left.is_empty()); + assert_eq!(left.len(), 3); + assert_eq!(left.slots.len(), 1); + assert_eq!(left.accounts.len(), 1); + assert_eq!(left.purged.len(), 1); + // Concatenation preserves the merged-in slot value. + assert_eq!(left.slots[0].new, U256::from(5)); + } + + #[test] + fn slot_delta_add_applies_saturating() { + assert_eq!( + SlotDelta::Add(U256::from(50)).apply(U256::from(100)), + U256::from(150) + ); + // Saturates at U256::MAX rather than wrapping. + assert_eq!( + SlotDelta::Add(U256::from(10)).apply(U256::MAX - U256::from(1)), + U256::MAX + ); + assert_eq!(SlotDelta::Add(U256::from(5)).apply(U256::MAX), U256::MAX); + } + + #[test] + fn slot_delta_sub_applies_saturating() { + assert_eq!( + SlotDelta::Sub(U256::from(30)).apply(U256::from(100)), + U256::from(70) + ); + // Saturates at zero rather than underflowing. + assert_eq!( + SlotDelta::Sub(U256::from(50)).apply(U256::from(30)), + U256::ZERO + ); + assert_eq!(SlotDelta::Sub(U256::from(1)).apply(U256::ZERO), U256::ZERO); + } + + #[test] + fn state_update_slot_delta_constructor() { + let a = addr(0xcc); + assert_eq!( + StateUpdate::slot_delta(a, U256::from(1), SlotDelta::Add(U256::from(2))), + StateUpdate::SlotDelta { + address: a, + slot: U256::from(1), + delta: SlotDelta::Add(U256::from(2)), + } + ); + } + + #[test] + fn state_diff_merge_extends_skipped_without_counting_it() { + let a = addr(0xdd); + let mut left = StateDiff::default(); + let mut right = StateDiff::default(); + right.skipped.push(SkippedDelta { + address: a, + slot: U256::from(1), + delta: SlotDelta::Sub(U256::from(3)), + }); + + left.merge(right); + assert_eq!(left.skipped.len(), 1); + // A skip is metadata, not a change. + assert!(left.is_empty()); + assert_eq!(left.len(), 0); + } +} diff --git a/tests/freshness.rs b/tests/freshness.rs index a892883..1c981a5 100644 --- a/tests/freshness.rs +++ b/tests/freshness.rs @@ -73,11 +73,18 @@ async fn verify_slots_detects_and_injects_changes() -> Result<()> { let slot_a = U256::from(10); let slot_b = U256::from(20); - // Cache holds these values. - cache.inject_storage_batch(&[ - (contract, slot_a, U256::from(100)), - (contract, slot_b, U256::from(200)), - ]); + // Cache holds these values, seeded OVERLAY-resident so they are EVM-visible: + // `contract` is a StorageCleared MockERC20, and after the §16.0 fix a + // backend-only `inject_storage_batch` seed on a StorageCleared account is + // shadowed to ZERO by `cached_storage_value` (it mirrors the EVM SLOAD). The + // test's intent is that the cache *holds* these values, so seed the layer that + // actually wins (mirrors `state_update::balance_tracking_scenario`). + cache + .db_mut() + .insert_account_storage(contract, slot_a, U256::from(100))?; + cache + .db_mut() + .insert_account_storage(contract, slot_b, U256::from(200))?; // Stub reports slot_a changed, slot_b unchanged. let values = HashMap::from([ @@ -115,7 +122,13 @@ async fn verify_slots_unchanged_returns_empty() -> Result<()> { install_mock_erc20(&mut cache, contract); let slot = U256::from(7); - cache.inject_storage_batch(&[(contract, slot, U256::from(42))]); + // Overlay-resident seed so the value is EVM-visible (see the note in + // `verify_slots_detects_and_injects_changes`): a backend-only seed on this + // StorageCleared MockERC20 would read as ZERO under the §16.0 fix, so the + // fetcher's matching 42 would (incorrectly) look like a 0 -> 42 change. + cache + .db_mut() + .insert_account_storage(contract, slot, U256::from(42))?; cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([( (contract, slot), U256::from(42), @@ -547,7 +560,15 @@ async fn run_drains_pending_on_next_run() -> Result<()> { install_default_account(&mut cache, Address::ZERO); install_default_account(&mut cache, owner); install_mock_erc20(&mut cache, token); - cache.inject_storage_batch(&[(token, balance_slot_for(owner), U256::from(1000))]); + // Overlay-resident seed so the balance is EVM-visible on the StorageCleared + // token account (see the note in `verify_slots_detects_and_injects_changes`): + // after the §16.0 fix, a backend-only `inject_storage_batch` seed here reads as + // ZERO via `cached_storage_value` (mirroring the SLOAD), so the live-cache + // assertions below would observe 0 instead of the seeded value. This mirrors + // `state_update::balance_tracking_scenario`. + cache + .db_mut() + .insert_account_storage(token, balance_slot_for(owner), U256::from(1000))?; cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([( (token, balance_slot_for(owner)), U256::from(2000), @@ -1036,11 +1057,17 @@ async fn validator_fetches_at_snapshot_block_despite_repin() -> Result<()> { install_default_account(&mut cache, owner); install_mock_erc20(&mut cache, token); cache.set_block(Some(block_n)); - cache.inject_storage_batch(&[(token, slot, U256::from(1000))]); + // Overlay-resident seed so the balance is EVM-visible on the StorageCleared + // token account (see the note in `verify_slots_detects_and_injects_changes`): + // a backend-only seed would read as ZERO under the §16.0 `cached_storage_value` + // fix, failing the precondition below. + cache + .db_mut() + .insert_account_storage(token, slot, U256::from(1000))?; assert_eq!( cache.cached_storage_value(token, slot), Some(U256::from(1000)), - "PRECONDITION: seeded balance present after set_block + inject" + "PRECONDITION: seeded balance present after set_block + insert" ); // Block-aware fetcher: the snapshot value (1000) at block N, a CHANGED value diff --git a/tests/snapshot_overlay.rs b/tests/snapshot_overlay.rs index 3b18abc..23a204d 100644 --- a/tests/snapshot_overlay.rs +++ b/tests/snapshot_overlay.rs @@ -101,7 +101,15 @@ async fn overlays_from_one_snapshot_are_isolated() -> Result<()> { let slot = U256::from(7); let original = U256::from(1u64); - cache.inject_storage_batch(&[(contract, slot, original)]); + // Overlay-resident seed so the value is EVM-visible on the StorageCleared + // MockERC20: after the §16.0 fix, a backend-only `inject_storage_batch` seed on + // a StorageCleared account reads as ZERO via `cached_storage_value` (mirroring + // the EVM SLOAD), so the live-cache assertion below would observe 0. Seeding + // the overlay (the winning layer) is what the test means by "the cache holds + // `original`" and is captured by `create_snapshot`. + cache + .db_mut() + .insert_account_storage(contract, slot, original)?; let snapshot = cache.create_snapshot(); let mut overlay_a = EvmOverlay::new(Arc::clone(&snapshot), None); diff --git a/tests/state_update.rs b/tests/state_update.rs new file mode 100644 index 0000000..5ff4225 --- /dev/null +++ b/tests/state_update.rs @@ -0,0 +1,1528 @@ +//! Offline acceptance tests for the Phase 3 state-update primitives (Pillar B.1). +//! +//! These are the **contract** the implementation must satisfy: the +//! `StateUpdate` vocabulary, `EvmCache::apply_update` / `apply_updates`, the +//! `StateDiff` output, and the refold of the existing writers. Everything runs +//! fully offline (mocked provider, state injected directly), so no test reaches +//! the network. +//! +//! Layering vocabulary used throughout: +//! - **layer 1 / overlay** = the CacheDB overlay (`db_mut().cache.accounts`), +//! which wins on reads. +//! - **layer 2 / backend** = the BlockchainDb backend +//! (`blockchain_db().storage()` / `.accounts()`). + +mod common; + +use alloy_primitives::{Address, Bytes, U256}; +use anyhow::Result; + +use common::{ + MOCK_ERC20_BALANCE_SLOT, balance_of, install_default_account, install_mock_erc20, setup_cache, +}; +use evm_fork_cache::cache::EvmCache; +use evm_fork_cache::{ + AccountPatch, PurgeScope, SkippedBalanceDelta, SkippedDelta, SlotChange, SlotDelta, StateDiff, + StateUpdate, +}; +use revm::state::{AccountInfo, Bytecode}; + +// --------------------------------------------------------------------------- +// Layer-inspection helpers (read each cache layer independently). +// --------------------------------------------------------------------------- + +/// Hashed storage slot of `balanceOf[owner]` for the MockERC20 fixture. +fn balance_slot_for(owner: Address) -> U256 { + use alloy_sol_types::SolValue; + let key = + alloy_primitives::keccak256((owner, U256::from(MOCK_ERC20_BALANCE_SLOT)).abi_encode()); + U256::from_be_bytes(key.0) +} + +/// Value of a slot in the CacheDB overlay (layer 1) only. +fn overlay_slot(cache: &mut EvmCache, addr: Address, slot: U256) -> Option { + cache + .db_mut() + .cache + .accounts + .get(&addr) + .and_then(|a| a.storage.get(&slot).copied()) +} + +/// Value of a slot in the BlockchainDb backend (layer 2) only. +fn backend_slot(cache: &EvmCache, addr: Address, slot: U256) -> Option { + cache + .blockchain_db() + .storage() + .read() + .get(&addr) + .and_then(|s| s.get(&slot).copied()) +} + +/// Whether the overlay (layer 1) has an account entry for `addr`. +fn overlay_has_account(cache: &mut EvmCache, addr: Address) -> bool { + cache.db_mut().cache.accounts.contains_key(&addr) +} + +/// Overlay (layer 1) balance for `addr`, if an overlay account exists. +fn overlay_balance(cache: &mut EvmCache, addr: Address) -> Option { + cache + .db_mut() + .cache + .accounts + .get(&addr) + .map(|a| a.info.balance) +} + +/// Overlay (layer 1) nonce for `addr`, if an overlay account exists. +fn overlay_nonce(cache: &mut EvmCache, addr: Address) -> Option { + cache + .db_mut() + .cache + .accounts + .get(&addr) + .map(|a| a.info.nonce) +} + +/// Backend (layer 2) balance for `addr`, if a backend account exists. +fn backend_balance(cache: &EvmCache, addr: Address) -> Option { + cache + .blockchain_db() + .accounts() + .read() + .get(&addr) + .map(|i| i.balance) +} + +// =========================================================================== +// Pure-data vocabulary (public API, no cache). +// =========================================================================== + +#[test] +fn account_patch_builders_compose() { + let empty = AccountPatch::default(); + assert_eq!(empty.balance, None); + assert_eq!(empty.nonce, None); + assert_eq!(empty.code, None); + + let patch = AccountPatch::default() + .balance(U256::from(42)) + .nonce(7) + .code(Bytes::from_static(&[0x60, 0x00])); + assert_eq!(patch.balance, Some(U256::from(42))); + assert_eq!(patch.nonce, Some(7)); + assert_eq!(patch.code, Some(Bytes::from_static(&[0x60, 0x00]))); +} + +#[test] +fn state_update_constructors_produce_expected_variants() { + let a = Address::repeat_byte(0xaa); + + assert_eq!( + StateUpdate::slot(a, U256::from(1), U256::from(2)), + StateUpdate::Slot { + address: a, + slot: U256::from(1), + value: U256::from(2), + } + ); + assert_eq!( + StateUpdate::balance(a, U256::from(9)), + StateUpdate::Account { + address: a, + patch: AccountPatch::default().balance(U256::from(9)), + } + ); + assert_eq!( + StateUpdate::purge(a, PurgeScope::Account), + StateUpdate::Purge { + address: a, + scope: PurgeScope::Account, + } + ); +} + +#[test] +fn state_diff_merge_and_is_empty() { + let a = Address::repeat_byte(0xbb); + let mut left = StateDiff::default(); + assert!(left.is_empty()); + assert_eq!(left.len(), 0); + + let mut right = StateDiff::default(); + right.slots.push(SlotChange { + address: a, + slot: U256::from(1), + old: U256::ZERO, + new: U256::from(5), + }); + + left.merge(right); + assert!(!left.is_empty()); + assert_eq!(left.len(), 1); + assert_eq!(left.slots.len(), 1); + assert_eq!(left.slots[0].new, U256::from(5)); +} + +// =========================================================================== +// Slot updates — write-through semantics (mirror inject_storage_batch_fresh). +// =========================================================================== + +#[tokio::test] +async fn apply_slot_writes_through_overlay_resident() -> Result<()> { + // An overlay-resident slot must be healed in BOTH layers, and the change + // must be observable on the synchronous EVM SLOAD path (here a balanceOf + // against a StorageCleared MockERC20 account). + let token = Address::repeat_byte(0x11); + let owner = Address::repeat_byte(0x22); + + let mut cache = setup_cache().await?; + install_default_account(&mut cache, Address::ZERO); // coinbase, for call_raw + install_default_account(&mut cache, owner); + install_mock_erc20(&mut cache, token); + + let slot = balance_slot_for(owner); + cache + .db_mut() + .insert_account_storage(token, slot, U256::from(100))?; + + let diff = cache.apply_update(&StateUpdate::slot(token, slot, U256::from(999))); + + // Both layers reflect the new value. + assert_eq!(overlay_slot(&mut cache, token, slot), Some(U256::from(999))); + assert_eq!(backend_slot(&cache, token, slot), Some(U256::from(999))); + assert_eq!( + cache.cached_storage_value(token, slot), + Some(U256::from(999)) + ); + + // The diff records exactly the one change. + assert_eq!( + diff.slots, + vec![SlotChange { + address: token, + slot, + old: U256::from(100), + new: U256::from(999), + }] + ); + assert!(diff.accounts.is_empty() && diff.purged.is_empty()); + + // The EVM SLOAD path sees the healed value. + assert_eq!(balance_of(&mut cache, token, owner)?, U256::from(999)); + Ok(()) +} + +#[tokio::test] +async fn apply_slot_no_overlay_account_is_not_materialized() -> Result<()> { + // Writing to an address with no overlay entry must populate the backend + // (layer 2) and NOT materialize a layer-1 overlay account — preserving the + // cold-prefetch / layer-2-only invariant. + let addr = Address::repeat_byte(0x33); + let slot = U256::from(7); + + let mut cache = setup_cache().await?; + assert!( + !overlay_has_account(&mut cache, addr), + "precondition: no overlay account" + ); + + let diff = cache.apply_update(&StateUpdate::slot(addr, slot, U256::from(5))); + + assert_eq!(backend_slot(&cache, addr, slot), Some(U256::from(5))); + assert!( + !overlay_has_account(&mut cache, addr), + "no overlay account may be materialized for a layer-2-only slot write" + ); + // The read falls through to the backend. + assert_eq!(cache.cached_storage_value(addr, slot), Some(U256::from(5))); + assert_eq!( + diff.slots, + vec![SlotChange { + address: addr, + slot, + old: U256::ZERO, + new: U256::from(5), + }] + ); + Ok(()) +} + +#[tokio::test] +async fn apply_slot_unchanged_value_yields_empty_diff() -> Result<()> { + let addr = Address::repeat_byte(0x44); + let slot = U256::from(1); + + let mut cache = setup_cache().await?; + // Seed the backend (layer 2) directly — inject_storage_batch does not load + // an account, unlike insert_account_storage, which would fetch a fresh + // address from the (mocked, empty) provider. + cache.inject_storage_batch(&[(addr, slot, U256::from(50))]); + + let diff = cache.apply_update(&StateUpdate::slot(addr, slot, U256::from(50))); + assert!( + diff.is_empty(), + "writing the cached value records no change" + ); + Ok(()) +} + +#[tokio::test] +async fn apply_slot_is_idempotent() -> Result<()> { + let addr = Address::repeat_byte(0x55); + let slot = U256::from(2); + + let mut cache = setup_cache().await?; + // Backend-direct seed (no account load) — see the note in the no-op test. + cache.inject_storage_batch(&[(addr, slot, U256::from(1))]); + + let first = cache.apply_update(&StateUpdate::slot(addr, slot, U256::from(8))); + assert_eq!(first.slots.len(), 1, "first apply records the change"); + + let second = cache.apply_update(&StateUpdate::slot(addr, slot, U256::from(8))); + assert!(second.is_empty(), "re-applying the same value is a no-op"); + Ok(()) +} + +// =========================================================================== +// Account updates — partial patch, write-through. +// =========================================================================== + +#[tokio::test] +async fn apply_account_balance_patch_preserves_other_fields() -> Result<()> { + let token = Address::repeat_byte(0x66); + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, token); // balance 0, nonce 0, code present + + let diff = cache.apply_update(&StateUpdate::Account { + address: token, + patch: AccountPatch::default().balance(U256::from(500)), + }); + + // Balance changed in the overlay (the winning layer); nonce/code preserved. + assert_eq!(overlay_balance(&mut cache, token), Some(U256::from(500))); + assert_eq!(overlay_nonce(&mut cache, token), Some(0)); + + assert_eq!(diff.accounts.len(), 1); + let change = &diff.accounts[0]; + assert_eq!(change.address, token); + assert_eq!(change.balance, Some((U256::ZERO, U256::from(500)))); + assert_eq!(change.nonce, None, "nonce unchanged → no delta"); + assert_eq!(change.code_hash, None, "code unchanged → no delta"); + assert!(diff.slots.is_empty() && diff.purged.is_empty()); + Ok(()) +} + +#[tokio::test] +async fn apply_account_code_patch_recomputes_hash() -> Result<()> { + let addr = Address::repeat_byte(0x77); + let mut cache = setup_cache().await?; + install_default_account(&mut cache, addr); // empty code + + let new_code = Bytes::from_static(&[0x60, 0x00, 0x60, 0x00, 0xf3]); + let expected_hash = Bytecode::new_raw(new_code.clone()).hash_slow(); + + let diff = cache.apply_update(&StateUpdate::Account { + address: addr, + patch: AccountPatch::default().code(new_code.clone()), + }); + + assert_eq!(diff.accounts.len(), 1); + let change = &diff.accounts[0]; + let (old_hash, new_hash) = change.code_hash.expect("code hash changed"); + assert_ne!(old_hash, new_hash); + assert_eq!( + new_hash, expected_hash, + "code hash recomputed from the patched code" + ); + assert_eq!(change.balance, None); + assert_eq!(change.nonce, None); + Ok(()) +} + +#[tokio::test] +async fn apply_account_patch_materializes_absent_account() -> Result<()> { + // An account absent from both layers is created (in the backend) by a patch, + // and the value is readable. + let addr = Address::repeat_byte(0x88); + let mut cache = setup_cache().await?; + assert!(!overlay_has_account(&mut cache, addr)); + assert_eq!(backend_balance(&cache, addr), None); + + let diff = cache.apply_update(&StateUpdate::balance(addr, U256::from(1234))); + + assert_eq!(backend_balance(&cache, addr), Some(U256::from(1234))); + assert_eq!(diff.accounts.len(), 1); + assert_eq!( + diff.accounts[0].balance, + Some((U256::ZERO, U256::from(1234))) + ); + Ok(()) +} + +// =========================================================================== +// Purge updates — dispatch to the existing layer logic, record what was removed. +// =========================================================================== + +#[tokio::test] +async fn apply_purge_account_clears_both_layers() -> Result<()> { + let token = Address::repeat_byte(0x11); + let owner = Address::repeat_byte(0x22); + + let mut cache = setup_cache().await?; + install_default_account(&mut cache, Address::ZERO); + install_default_account(&mut cache, owner); + install_mock_erc20(&mut cache, token); + + // Populate both layers. + common::transfer(&mut cache, token, owner, owner, U256::from(0)).ok(); + cache + .db_mut() + .insert_account_storage(token, U256::from(1), U256::from(9))?; + cache.inject_storage_batch(&[(token, U256::from(2), U256::from(8))]); + assert!(overlay_has_account(&mut cache, token)); + + let diff = cache.apply_update(&StateUpdate::purge(token, PurgeScope::Account)); + + assert!( + !overlay_has_account(&mut cache, token), + "overlay account removed" + ); + assert_eq!( + cache.pool_storage_slot_count(token), + 0, + "backend storage gone" + ); + { + let accounts = cache.blockchain_db().accounts().read(); + assert!(!accounts.contains_key(&token), "backend account removed"); + } + assert_eq!(diff.purged.len(), 1); + assert_eq!(diff.purged[0].address, token); + assert_eq!(diff.purged[0].scope, PurgeScope::Account); + assert!( + diff.purged[0].account_removed, + "an account info was removed" + ); + Ok(()) +} + +#[tokio::test] +async fn apply_purge_all_storage_keeps_account() -> Result<()> { + let token = Address::repeat_byte(0x33); + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, token); + cache + .db_mut() + .insert_account_storage(token, U256::from(1), U256::from(9))?; + cache.inject_storage_batch(&[(token, U256::from(2), U256::from(8))]); + + let diff = cache.apply_update(&StateUpdate::purge(token, PurgeScope::AllStorage)); + + assert_eq!( + cache.pool_storage_slot_count(token), + 0, + "backend storage gone" + ); + assert!( + overlay_has_account(&mut cache, token), + "account info preserved" + ); + assert_eq!(diff.purged.len(), 1); + assert_eq!(diff.purged[0].scope, PurgeScope::AllStorage); + assert!(!diff.purged[0].account_removed); + Ok(()) +} + +#[tokio::test] +async fn apply_purge_specific_slots() -> Result<()> { + let token = Address::repeat_byte(0x44); + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, token); + cache.inject_storage_batch(&[ + (token, U256::from(1), U256::from(10)), + (token, U256::from(2), U256::from(20)), + (token, U256::from(3), U256::from(30)), + ]); + + let diff = cache.apply_update(&StateUpdate::purge( + token, + PurgeScope::Slots(vec![U256::from(1), U256::from(3)]), + )); + + assert_eq!(backend_slot(&cache, token, U256::from(1)), None); + assert_eq!( + backend_slot(&cache, token, U256::from(2)), + Some(U256::from(20)) + ); + assert_eq!(backend_slot(&cache, token, U256::from(3)), None); + assert_eq!(diff.purged.len(), 1); + assert_eq!(diff.purged[0].slots_removed, 2); + Ok(()) +} + +// =========================================================================== +// apply_updates — fold + merge. +// =========================================================================== + +#[tokio::test] +async fn apply_updates_merges_mixed_batch() -> Result<()> { + let acct = Address::repeat_byte(0x66); + let pool = Address::repeat_byte(0x77); + + let mut cache = setup_cache().await?; + install_default_account(&mut cache, acct); + cache.inject_storage_batch(&[(pool, U256::from(9), U256::from(1))]); + + let diff = cache.apply_updates(&[ + StateUpdate::slot(pool, U256::from(1), U256::from(100)), + StateUpdate::balance(acct, U256::from(500)), + StateUpdate::purge(pool, PurgeScope::Slots(vec![U256::from(9)])), + ]); + + assert!(!diff.slots.is_empty(), "slot write recorded"); + assert!(!diff.accounts.is_empty(), "account patch recorded"); + assert!(!diff.purged.is_empty(), "purge recorded"); + Ok(()) +} + +#[tokio::test] +async fn apply_updates_same_slot_later_overrides() -> Result<()> { + let addr = Address::repeat_byte(0x88); + let slot = U256::from(1); + let mut cache = setup_cache().await?; + + let diff = cache.apply_updates(&[ + StateUpdate::slot(addr, slot, U256::from(10)), + StateUpdate::slot(addr, slot, U256::from(20)), + ]); + + // Each apply contributes its own SlotChange (merge concatenates), so the + // observed history is ZERO->10 then 10->20. + assert_eq!( + diff.slots, + vec![ + SlotChange { + address: addr, + slot, + old: U256::ZERO, + new: U256::from(10) + }, + SlotChange { + address: addr, + slot, + old: U256::from(10), + new: U256::from(20) + }, + ] + ); + assert_eq!(cache.cached_storage_value(addr, slot), Some(U256::from(20))); + Ok(()) +} + +// =========================================================================== +// Refold equivalence — wrappers behave exactly as before. +// =========================================================================== + +#[tokio::test] +async fn refold_purge_pool_storage_returns_same_count() -> Result<()> { + let token = Address::repeat_byte(0x99); + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, token); + cache.inject_storage_batch(&[ + (token, U256::from(1), U256::from(10)), + (token, U256::from(2), U256::from(20)), + ]); + + // The wrapper still returns the backend slot count it removed. + let removed = cache.purge_pool_storage(token); + assert_eq!(removed, 2); + assert_eq!(cache.pool_storage_slot_count(token), 0); + Ok(()) +} + +#[tokio::test] +async fn refold_inject_storage_batch_fresh_matches_apply_updates() -> Result<()> { + let token = Address::repeat_byte(0xa1); + let slot = U256::from(4); + + // Path A: the existing wrapper. + let mut a = setup_cache().await?; + install_mock_erc20(&mut a, token); + a.db_mut() + .insert_account_storage(token, slot, U256::from(1))?; + a.inject_storage_batch_fresh(&[(token, slot, U256::from(77))]); + + // Path B: the primitive it now wraps. + let mut b = setup_cache().await?; + install_mock_erc20(&mut b, token); + b.db_mut() + .insert_account_storage(token, slot, U256::from(1))?; + let _ = b.apply_updates(&[StateUpdate::slot(token, slot, U256::from(77))]); + + assert_eq!( + a.cached_storage_value(token, slot), + b.cached_storage_value(token, slot), + "wrapper and primitive leave the cache in the same state" + ); + assert_eq!( + overlay_slot(&mut a, token, slot), + overlay_slot(&mut b, token, slot) + ); + assert_eq!(backend_slot(&a, token, slot), backend_slot(&b, token, slot)); + Ok(()) +} + +// =========================================================================== +// Decision 2 (LOCKED: normalize) — protocols inject_v3_* now writes through to +// the backend (layer 2). Pre-fix this wrote layer 1 only. +// =========================================================================== + +#[cfg(feature = "protocols")] +#[tokio::test] +async fn inject_v3_tick_bitmap_writes_through_to_backend() -> Result<()> { + use std::collections::HashMap; + + let pool = Address::repeat_byte(0xb2); + let mut cache = setup_cache().await?; + + let mut bitmap = HashMap::new(); + bitmap.insert(0i16, U256::from(123)); + bitmap.insert(1i16, U256::from(456)); + + let injected = cache.inject_v3_tick_bitmap(pool, &bitmap)?; + assert_eq!(injected, 2); + + // Normalized to write-through: the backend (layer 2) now holds the slots. + // Before the refold this count was 0 (overlay-only write). + assert!( + cache.pool_storage_slot_count(pool) > 0, + "inject_v3_tick_bitmap must write through to the backend (Decision 2)" + ); + Ok(()) +} + +// =========================================================================== +// §15 addendum — relative / read-modify-write updates. +// +// `SlotDelta` reads the current value, applies a saturating mutation, and writes +// back (write-through). It is cold-aware: a delta on a slot the cache never +// fetched is NOT applied (it would corrupt an unknown balance) — it is skipped +// and surfaced in `StateDiff.skipped`. `modify_slot` is the general closure form. +// =========================================================================== + +#[tokio::test] +async fn slot_delta_add_applies_to_hot_slot() -> Result<()> { + let addr = Address::repeat_byte(0xc1); + let slot = U256::from(1); + let mut cache = setup_cache().await?; + cache.inject_storage_batch(&[(addr, slot, U256::from(100))]); + + let diff = cache.apply_update(&StateUpdate::slot_delta( + addr, + slot, + SlotDelta::Add(U256::from(50)), + )); + + assert_eq!( + cache.cached_storage_value(addr, slot), + Some(U256::from(150)) + ); + assert_eq!( + diff.slots, + vec![SlotChange { + address: addr, + slot, + old: U256::from(100), + new: U256::from(150), + }] + ); + assert!( + diff.skipped.is_empty(), + "a hot slot is applied, not skipped" + ); + Ok(()) +} + +#[tokio::test] +async fn slot_delta_sub_saturates_at_zero() -> Result<()> { + let addr = Address::repeat_byte(0xc2); + let slot = U256::from(1); + let mut cache = setup_cache().await?; + cache.inject_storage_batch(&[(addr, slot, U256::from(30))]); + + let diff = cache.apply_update(&StateUpdate::slot_delta( + addr, + slot, + SlotDelta::Sub(U256::from(50)), + )); + + assert_eq!( + cache.cached_storage_value(addr, slot), + Some(U256::ZERO), + "Sub saturates at zero rather than underflowing" + ); + assert_eq!(diff.slots.len(), 1); + assert_eq!(diff.slots[0].new, U256::ZERO); + Ok(()) +} + +#[tokio::test] +async fn slot_delta_add_saturates_at_max() -> Result<()> { + let addr = Address::repeat_byte(0xc3); + let slot = U256::from(1); + let mut cache = setup_cache().await?; + cache.inject_storage_batch(&[(addr, slot, U256::MAX - U256::from(1))]); + + cache.apply_update(&StateUpdate::slot_delta( + addr, + slot, + SlotDelta::Add(U256::from(10)), + )); + + assert_eq!( + cache.cached_storage_value(addr, slot), + Some(U256::MAX), + "Add saturates at U256::MAX" + ); + Ok(()) +} + +#[tokio::test] +async fn slot_delta_cold_slot_is_skipped_and_surfaced() -> Result<()> { + // The correctness guarantee: a delta against an unknown (cold) value must not + // be applied (it would corrupt the balance) — it is surfaced instead. + let addr = Address::repeat_byte(0xc4); + let slot = U256::from(7); + let mut cache = setup_cache().await?; + assert_eq!( + cache.cached_storage_value(addr, slot), + None, + "precondition: slot is cold" + ); + + let diff = cache.apply_update(&StateUpdate::slot_delta( + addr, + slot, + SlotDelta::Add(U256::from(50)), + )); + + assert!(diff.slots.is_empty(), "nothing applied"); + assert_eq!( + diff.skipped, + vec![SkippedDelta { + address: addr, + slot, + delta: SlotDelta::Add(U256::from(50)), + }] + ); + assert_eq!( + cache.cached_storage_value(addr, slot), + None, + "the cold slot is left untouched so the next read fetches the truth" + ); + Ok(()) +} + +#[tokio::test] +async fn slot_delta_writes_through_both_layers() -> Result<()> { + let token = Address::repeat_byte(0xc5); + let slot = U256::from(2); + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, token); + // Overlay-resident seed (account already installed, so no fetch). + cache + .db_mut() + .insert_account_storage(token, slot, U256::from(100))?; + + cache.apply_update(&StateUpdate::slot_delta( + token, + slot, + SlotDelta::Add(U256::from(5)), + )); + + assert_eq!(overlay_slot(&mut cache, token, slot), Some(U256::from(105))); + assert_eq!(backend_slot(&cache, token, slot), Some(U256::from(105))); + Ok(()) +} + +#[tokio::test] +async fn modify_slot_applies_transform() -> Result<()> { + let addr = Address::repeat_byte(0xc6); + let slot = U256::from(1); + let mut cache = setup_cache().await?; + cache.inject_storage_batch(&[(addr, slot, U256::from(10))]); + + let change = cache.modify_slot(addr, slot, |cur| cur.map(|v| v * U256::from(2))); + + assert_eq!( + change, + Some(SlotChange { + address: addr, + slot, + old: U256::from(10), + new: U256::from(20), + }) + ); + assert_eq!(cache.cached_storage_value(addr, slot), Some(U256::from(20))); + Ok(()) +} + +#[tokio::test] +async fn modify_slot_closure_skips_cold() -> Result<()> { + let addr = Address::repeat_byte(0xc7); + let slot = U256::from(1); + let mut cache = setup_cache().await?; + + // The closure returns None for a cold slot, so nothing is written. + let change = cache.modify_slot(addr, slot, |cur| cur.map(|v| v + U256::from(1))); + + assert_eq!(change, None); + assert_eq!(cache.cached_storage_value(addr, slot), None); + Ok(()) +} + +#[tokio::test] +async fn modify_slot_can_write_absolute_on_cold() -> Result<()> { + // The caller may choose to write an absolute value even on a cold slot (it + // had external knowledge). The closure ignores the `None` and returns a value. + let addr = Address::repeat_byte(0xc8); + let slot = U256::from(1); + let mut cache = setup_cache().await?; + + let change = cache.modify_slot(addr, slot, |_| Some(U256::from(7))); + + assert_eq!( + change, + Some(SlotChange { + address: addr, + slot, + old: U256::ZERO, + new: U256::from(7), + }) + ); + assert_eq!(cache.cached_storage_value(addr, slot), Some(U256::from(7))); + Ok(()) +} + +#[test] +fn state_diff_merge_includes_skipped() { + let a = Address::repeat_byte(0xd9); + let mut left = StateDiff::default(); + let mut right = StateDiff::default(); + right.skipped.push(SkippedDelta { + address: a, + slot: U256::from(1), + delta: SlotDelta::Add(U256::from(5)), + }); + + left.merge(right); + assert_eq!(left.skipped.len(), 1); + assert_eq!(left.skipped[0].delta, SlotDelta::Add(U256::from(5))); + // A skip is metadata, not a change: it does not affect is_empty/len. + assert!(left.is_empty(), "a skipped delta is not a recorded change"); + assert_eq!(left.len(), 0); +} + +#[tokio::test] +async fn balance_tracking_scenario() -> Result<()> { + // The motivating use case: index an ERC-20 `Transfer(alice -> bob, amount)` + // as two relative slot updates to keep the tracked balances hot, without ever + // knowing the resulting absolute balances up front. + let token = Address::repeat_byte(0xe0); + let alice = Address::repeat_byte(0x0a); + let bob = Address::repeat_byte(0x0b); + let mut cache = setup_cache().await?; + install_default_account(&mut cache, Address::ZERO); // coinbase, for the SLOAD calls + install_default_account(&mut cache, alice); + install_default_account(&mut cache, bob); + install_mock_erc20(&mut cache, token); + + let alice_slot = balance_slot_for(alice); + let bob_slot = balance_slot_for(bob); + + // Seed the tracked balances once (the "make it hot" step) in an EVM-VISIBLE + // way: overlay-resident, so the StorageCleared token account actually reads + // them on the SLOAD path. (A backend-only inject is invisible here — see + // `cached_storage_value_matches_evm_sload_for_cleared_account`.) + cache + .db_mut() + .insert_account_storage(token, alice_slot, U256::from(1000))?; + cache + .db_mut() + .insert_account_storage(token, bob_slot, U256::ZERO)?; + + // Sanity: the EVM actually sees the seeded balances. + assert_eq!(balance_of(&mut cache, token, alice)?, U256::from(1000)); + + // Transfer(alice -> bob, 300) decodes to two relative updates. + let amount = U256::from(300); + let diff = cache.apply_updates(&[ + StateUpdate::slot_delta(token, alice_slot, SlotDelta::Sub(amount)), + StateUpdate::slot_delta(token, bob_slot, SlotDelta::Add(amount)), + ]); + + // Validate via a real SLOAD (`balanceOf`), not just the cached accessor. + assert_eq!(balance_of(&mut cache, token, alice)?, U256::from(700)); + assert_eq!(balance_of(&mut cache, token, bob)?, U256::from(300)); + assert_eq!( + cache.cached_storage_value(token, alice_slot), + Some(U256::from(700)) + ); + assert_eq!( + cache.cached_storage_value(token, bob_slot), + Some(U256::from(300)) + ); + assert!(diff.skipped.is_empty(), "both slots were seeded (hot)"); + assert_eq!(diff.slots.len(), 2); + + // Conservation: total supply across the two holders is unchanged. + let total = cache.cached_storage_value(token, alice_slot).unwrap() + + cache.cached_storage_value(token, bob_slot).unwrap(); + assert_eq!(total, U256::from(1000)); + Ok(()) +} + +// =========================================================================== +// §16.0 — the audit HIGH correctness bug: cached_storage_value must match the +// EVM SLOAD for a StorageCleared overlay account (else SlotDelta corrupts an +// EVM-invisible base). This test uses only existing symbols so it runs against +// the CURRENT (buggy) code: it is RED before the §16.0 fix, GREEN after. +// =========================================================================== + +#[tokio::test] +async fn cached_storage_value_matches_evm_sload_for_cleared_account() -> Result<()> { + let token = Address::repeat_byte(0x5c); + let owner = Address::repeat_byte(0x5d); + let mut cache = setup_cache().await?; + install_default_account(&mut cache, Address::ZERO); // coinbase, for call_raw + install_default_account(&mut cache, owner); + install_mock_erc20(&mut cache, token); // account_state = StorageCleared + + let slot = balance_slot_for(owner); + // Backend-only seed: invisible to a StorageCleared overlay account's SLOAD. + cache.inject_storage_batch(&[(token, slot, U256::from(100))]); + + // The real EVM SLOAD reads ZERO (StorageCleared, slot absent from overlay, + // backend NOT consulted). + let evm_seen = balance_of(&mut cache, token, owner)?; + assert_eq!( + evm_seen, + U256::ZERO, + "precondition: the EVM cannot see a backend-only seed on a StorageCleared account" + ); + + // cached_storage_value MUST agree with the EVM, not report the shadowed + // backend value (100). Pre-fix it returns Some(100) -> this assert fails. + assert_eq!( + cache.cached_storage_value(token, slot), + Some(U256::ZERO), + "cached_storage_value must mirror the EVM SLOAD (ZERO), not the shadowed backend value" + ); + Ok(()) +} + +// =========================================================================== +// §16.0 — present-as-ZERO is HOT (delta applies to 0), distinct from cold (skip). +// =========================================================================== + +#[tokio::test] +async fn slot_delta_on_present_zero_is_hot_not_skipped() -> Result<()> { + let token = Address::repeat_byte(0x6a); + let slot = U256::from(1); + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, token); + // Overlay-resident ZERO: a *known* zero, not an absent (cold) slot. + cache + .db_mut() + .insert_account_storage(token, slot, U256::ZERO)?; + + let diff = cache.apply_update(&StateUpdate::slot_delta( + token, + slot, + SlotDelta::Add(U256::from(50)), + )); + + assert_eq!( + cache.cached_storage_value(token, slot), + Some(U256::from(50)) + ); + assert_eq!( + diff.slots.len(), + 1, + "present-as-zero is hot: the delta applies" + ); + assert!( + diff.skipped.is_empty(), + "present-as-zero must NOT be treated as cold" + ); + Ok(()) +} + +// =========================================================================== +// §16.5 — account-native-balance delta (BalanceDelta + modify_account_balance). +// =========================================================================== + +#[tokio::test] +async fn balance_delta_applies_to_present_account() -> Result<()> { + let acct = Address::repeat_byte(0x71); + let mut cache = setup_cache().await?; + cache.db_mut().insert_account_info( + acct, + AccountInfo { + balance: U256::from(1000), + nonce: 5, + ..Default::default() + }, + ); + + let diff = cache.apply_update(&StateUpdate::balance_delta( + acct, + SlotDelta::Sub(U256::from(300)), + )); + + assert_eq!(overlay_balance(&mut cache, acct), Some(U256::from(700))); + assert_eq!(overlay_nonce(&mut cache, acct), Some(5), "nonce preserved"); + assert_eq!( + backend_balance(&cache, acct), + Some(U256::from(700)), + "write-through to backend" + ); + assert_eq!(diff.accounts.len(), 1); + assert_eq!( + diff.accounts[0].balance, + Some((U256::from(1000), U256::from(700))) + ); + assert!(diff.accounts[0].nonce.is_none(), "nonce unchanged"); + assert!(diff.skipped_balances.is_empty()); + Ok(()) +} + +#[tokio::test] +async fn balance_delta_on_cold_account_is_skipped_and_surfaced() -> Result<()> { + let acct = Address::repeat_byte(0x72); + let mut cache = setup_cache().await?; + assert!(!overlay_has_account(&mut cache, acct)); + + let diff = cache.apply_update(&StateUpdate::balance_delta( + acct, + SlotDelta::Add(U256::from(500)), + )); + + assert!( + diff.accounts.is_empty(), + "nothing applied for an unknown balance" + ); + assert_eq!( + diff.skipped_balances, + vec![SkippedBalanceDelta { + address: acct, + delta: SlotDelta::Add(U256::from(500)), + }] + ); + // Crucially: no account is materialized (avoids masking the real on-chain one). + assert!(!overlay_has_account(&mut cache, acct)); + assert_eq!(backend_balance(&cache, acct), None); + Ok(()) +} + +#[tokio::test] +async fn balance_delta_saturates_at_zero() -> Result<()> { + let acct = Address::repeat_byte(0x73); + let mut cache = setup_cache().await?; + cache.db_mut().insert_account_info( + acct, + AccountInfo { + balance: U256::from(100), + ..Default::default() + }, + ); + + cache.apply_update(&StateUpdate::balance_delta( + acct, + SlotDelta::Sub(U256::from(500)), + )); + + assert_eq!( + overlay_balance(&mut cache, acct), + Some(U256::ZERO), + "Sub saturates at zero" + ); + Ok(()) +} + +#[tokio::test] +async fn modify_account_balance_hot_and_cold() -> Result<()> { + let acct = Address::repeat_byte(0x74); + let mut cache = setup_cache().await?; + cache.db_mut().insert_account_info( + acct, + AccountInfo { + balance: U256::from(10), + ..Default::default() + }, + ); + + let change = cache.modify_account_balance(acct, |cur| cur.map(|v| v * U256::from(3))); + assert_eq!( + change.and_then(|c| c.balance), + Some((U256::from(10), U256::from(30))) + ); + assert_eq!(overlay_balance(&mut cache, acct), Some(U256::from(30))); + + // A cold account: the closure receives None and skips; nothing materialized. + let cold = Address::repeat_byte(0x75); + let none = cache.modify_account_balance(cold, |cur| cur.map(|v| v + U256::from(1))); + assert!(none.is_none()); + assert!(!overlay_has_account(&mut cache, cold)); + assert_eq!(backend_balance(&cache, cold), None); + Ok(()) +} + +// =========================================================================== +// §16.6 — discoverable skip accessors over both skip kinds. +// =========================================================================== + +#[tokio::test] +async fn skip_accessors_reflect_both_skip_kinds() -> Result<()> { + let token = Address::repeat_byte(0x76); + let acct = Address::repeat_byte(0x77); + let mut cache = setup_cache().await?; + + // A cold slot delta and a cold balance delta: both skipped, no change recorded. + let diff = cache.apply_updates(&[ + StateUpdate::slot_delta(token, U256::from(9), SlotDelta::Add(U256::from(1))), + StateUpdate::balance_delta(acct, SlotDelta::Add(U256::from(1))), + ]); + assert!(diff.is_empty(), "changes-only: nothing applied"); + assert!(diff.has_skipped()); + assert_eq!(diff.skipped_len(), 2); + assert!(!diff.is_fully_applied()); + + // A fully-applied update reports no skips. + let hot = Address::repeat_byte(0x78); + cache.db_mut().insert_account_info( + hot, + AccountInfo { + balance: U256::from(5), + ..Default::default() + }, + ); + let diff2 = cache.apply_update(&StateUpdate::balance_delta( + hot, + SlotDelta::Add(U256::from(5)), + )); + assert!(diff2.is_fully_applied()); + assert!(!diff2.has_skipped()); + Ok(()) +} + +// =========================================================================== +// §16.3 — serde round-trip of the vocabulary and the diff. +// =========================================================================== + +#[test] +fn vocabulary_serde_round_trips() { + let a = Address::repeat_byte(0x81); + let updates = vec![ + StateUpdate::slot(a, U256::from(1), U256::from(2)), + StateUpdate::slot_delta(a, U256::from(1), SlotDelta::Sub(U256::from(3))), + StateUpdate::balance_delta(a, SlotDelta::Add(U256::from(4))), + StateUpdate::account(a, AccountPatch::default().balance(U256::from(9)).nonce(2)), + StateUpdate::purge(a, PurgeScope::Slots(vec![U256::from(1)])), + ]; + let json = serde_json::to_string(&updates).expect("serialize updates"); + let back: Vec = serde_json::from_str(&json).expect("deserialize updates"); + assert_eq!(updates, back); + + let mut diff = StateDiff::default(); + diff.slots.push(SlotChange { + address: a, + slot: U256::from(1), + old: U256::ZERO, + new: U256::from(2), + }); + diff.skipped.push(SkippedDelta { + address: a, + slot: U256::from(2), + delta: SlotDelta::Add(U256::from(1)), + }); + diff.skipped_balances.push(SkippedBalanceDelta { + address: a, + delta: SlotDelta::Sub(U256::from(1)), + }); + let djson = serde_json::to_string(&diff).expect("serialize diff"); + let dback: StateDiff = serde_json::from_str(&djson).expect("deserialize diff"); + assert_eq!(diff, dback); +} + +// =========================================================================== +// §16.1 — a no-op Account patch must not materialize a backend account. +// =========================================================================== + +#[tokio::test] +async fn account_patch_noop_does_not_materialize_backend() -> Result<()> { + let acct = Address::repeat_byte(0x95); + let mut cache = setup_cache().await?; + assert!(!overlay_has_account(&mut cache, acct)); + + // All-None patch on an absent account: no change, and crucially no write. + let diff = cache.apply_update(&StateUpdate::account(acct, AccountPatch::default())); + assert!(diff.is_empty()); + assert!(diff.accounts.is_empty()); + assert_eq!( + backend_balance(&cache, acct), + None, + "a no-op patch must not materialize a backend account" + ); + assert!(!overlay_has_account(&mut cache, acct)); + + // balance -> current value on a present account is also a no-op. + let acct2 = Address::repeat_byte(0x96); + cache.db_mut().insert_account_info( + acct2, + AccountInfo { + balance: U256::from(50), + ..Default::default() + }, + ); + let diff2 = cache.apply_update(&StateUpdate::balance(acct2, U256::from(50))); + assert!(diff2.accounts.is_empty(), "balance -> current is a no-op"); + Ok(()) +} + +// =========================================================================== +// §16.8/16.9 — batched apply_updates must equal sequential apply_update (the +// safety net for the single-lock fast-path). Mixed batch: distinct addresses, +// a same-slot repeat, a hot delta, an account patch, and a purge mid-batch. +// =========================================================================== + +#[tokio::test] +async fn apply_updates_batched_equals_sequential() -> Result<()> { + let p = Address::repeat_byte(0xa1); + let q = Address::repeat_byte(0xa2); + let slot1 = U256::from(1); + let slot2 = U256::from(2); + let slot3 = U256::from(3); + + // Build two identically-seeded caches. + async fn seeded(p: Address, q: Address, slot2: U256) -> Result { + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, p); // StorageCleared overlay account + install_default_account(&mut cache, q); + cache.inject_storage_batch(&[(p, slot2, U256::from(99))]); // backend slot for the purge + Ok(cache) + } + let batch = vec![ + StateUpdate::slot(p, slot1, U256::from(500)), + StateUpdate::slot(p, slot1, U256::from(600)), // same slot again (order matters) + StateUpdate::slot_delta(p, slot1, SlotDelta::Add(U256::from(10))), // hot -> 610 + StateUpdate::balance(q, U256::from(1000)), + StateUpdate::purge(p, PurgeScope::Slots(vec![slot2])), // purge mid-batch + StateUpdate::slot(p, slot3, U256::from(7)), // write after the purge + ]; + + let mut batched = seeded(p, q, slot2).await?; + let diff_batched = batched.apply_updates(&batch); + + let mut sequential = seeded(p, q, slot2).await?; + let mut diff_seq = StateDiff::default(); + for u in &batch { + diff_seq.merge(sequential.apply_update(u)); + } + + assert_eq!( + diff_batched, diff_seq, + "batched diff must equal the sequential fold" + ); + for (addr, slot) in [(p, slot1), (p, slot2), (p, slot3)] { + assert_eq!( + batched.cached_storage_value(addr, slot), + sequential.cached_storage_value(addr, slot), + "slot {slot} state diverged between batched and sequential" + ); + } + assert_eq!( + overlay_balance(&mut batched, q), + overlay_balance(&mut sequential, q) + ); + assert_eq!( + backend_balance(&batched, q), + backend_balance(&sequential, q) + ); + // Concrete expected end-state. + assert_eq!( + batched.cached_storage_value(p, slot1), + Some(U256::from(610)) + ); + // Verify the purge via the backend layer directly: `p` is a StorageCleared + // MockERC20, so cached_storage_value reads an absent slot as 0 (mirroring the + // SLOAD) regardless of the purge — the backend map is the meaningful check. + assert_eq!( + backend_slot(&batched, p, slot2), + None, + "slot2 purged from the backend" + ); + assert_eq!(batched.cached_storage_value(p, slot3), Some(U256::from(7))); + Ok(()) +} + +// =========================================================================== +// §16.8 — Account-patch coverage gaps. +// =========================================================================== + +#[tokio::test] +async fn account_patch_writes_through_to_backend_on_overlay_present() -> Result<()> { + let acct = Address::repeat_byte(0x90); + let mut cache = setup_cache().await?; + cache.db_mut().insert_account_info( + acct, + AccountInfo { + balance: U256::from(100), + ..Default::default() + }, + ); + + let diff = cache.apply_update(&StateUpdate::balance(acct, U256::from(500))); + + assert_eq!(overlay_balance(&mut cache, acct), Some(U256::from(500))); + assert_eq!( + backend_balance(&cache, acct), + Some(U256::from(500)), + "backend is always written, even when an overlay account exists" + ); + assert_eq!( + diff.accounts[0].balance, + Some((U256::from(100), U256::from(500))) + ); + Ok(()) +} + +#[tokio::test] +async fn account_patch_on_backend_only_account_does_not_materialize_overlay() -> Result<()> { + let acct = Address::repeat_byte(0x91); + let mut cache = setup_cache().await?; + // Seed only the backend (the cold-prefetched, layer-2-only case). + cache.blockchain_db().accounts().write().insert( + acct, + AccountInfo { + balance: U256::from(100), + nonce: 3, + ..Default::default() + }, + ); + + let diff = cache.apply_update(&StateUpdate::balance(acct, U256::from(500))); + + assert_eq!( + diff.accounts[0].balance, + Some((U256::from(100), U256::from(500))), + "old value loaded from the backend" + ); + assert_eq!(backend_balance(&cache, acct), Some(U256::from(500))); + assert!( + !overlay_has_account(&mut cache, acct), + "no overlay account materialized for a backend-only patch" + ); + Ok(()) +} + +#[tokio::test] +async fn account_patch_nonce_only() -> Result<()> { + let acct = Address::repeat_byte(0x92); + let mut cache = setup_cache().await?; + cache.db_mut().insert_account_info( + acct, + AccountInfo { + nonce: 1, + ..Default::default() + }, + ); + + let diff = cache.apply_update(&StateUpdate::nonce(acct, 9)); + + assert_eq!(diff.accounts[0].nonce, Some((1, 9))); + assert!(diff.accounts[0].balance.is_none()); + assert!(diff.accounts[0].code_hash.is_none()); + assert_eq!(overlay_nonce(&mut cache, acct), Some(9)); + Ok(()) +} + +#[tokio::test] +async fn account_patch_multi_field() -> Result<()> { + let acct = Address::repeat_byte(0x93); + let mut cache = setup_cache().await?; + cache + .db_mut() + .insert_account_info(acct, AccountInfo::default()); + + let diff = cache.apply_update(&StateUpdate::account( + acct, + AccountPatch::default() + .balance(U256::from(42)) + .nonce(7) + .code(Bytes::from_static(&[0x60, 0x00])), + )); + + assert!(diff.accounts[0].balance.is_some()); + assert!(diff.accounts[0].nonce.is_some()); + assert!(diff.accounts[0].code_hash.is_some()); + Ok(()) +} + +#[tokio::test] +async fn account_patch_empty_code_clears_to_empty_hash() -> Result<()> { + let token = Address::repeat_byte(0x94); + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, token); // non-empty code + + let diff = cache.apply_update(&StateUpdate::code(token, Bytes::new())); + let empty_hash = Bytecode::new_raw(Bytes::new()).hash_slow(); + assert_eq!( + diff.accounts[0].code_hash.expect("code changed").1, + empty_hash + ); + + // Patching empty over already-empty is a no-op. + let diff2 = cache.apply_update(&StateUpdate::code(token, Bytes::new())); + assert!(diff2.accounts.is_empty(), "empty over empty is a no-op"); + Ok(()) +} + +// =========================================================================== +// §16.8 — modify_slot write-through layer policy. +// =========================================================================== + +#[tokio::test] +async fn modify_slot_writes_through_both_layers() -> Result<()> { + let token = Address::repeat_byte(0x97); + let slot = U256::from(1); + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, token); + cache + .db_mut() + .insert_account_storage(token, slot, U256::from(10))?; + + cache.modify_slot(token, slot, |c| c.map(|v| v + U256::from(5))); + + assert_eq!(overlay_slot(&mut cache, token, slot), Some(U256::from(15))); + assert_eq!(backend_slot(&cache, token, slot), Some(U256::from(15))); + Ok(()) +} + +// =========================================================================== +// §16.8 — purge edges. +// =========================================================================== + +#[tokio::test] +async fn purge_absent_account_is_noop_record() -> Result<()> { + let acct = Address::repeat_byte(0x98); + let mut cache = setup_cache().await?; + + let diff = cache.apply_update(&StateUpdate::purge(acct, PurgeScope::Account)); + + assert_eq!(diff.purged.len(), 1); + assert!(!diff.purged[0].account_removed); + assert_eq!(diff.purged[0].slots_removed, 0); + Ok(()) +} + +#[tokio::test] +async fn purge_slots_counts_present_backend_slots_only() -> Result<()> { + let token = Address::repeat_byte(0x99); + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, token); + cache.inject_storage_batch(&[(token, U256::from(1), U256::from(10))]); // only slot 1 present + + let diff = cache.apply_update(&StateUpdate::purge( + token, + PurgeScope::Slots(vec![U256::from(1), U256::from(2)]), // slot 2 absent + )); + + assert_eq!( + diff.purged[0].slots_removed, 1, + "only the present backend slot is counted" + ); + Ok(()) +} + +// =========================================================================== +// §16.7 — account-field convenience constructors. +// =========================================================================== + +#[test] +fn state_update_account_field_constructors() { + let a = Address::repeat_byte(0x9a); + assert_eq!( + StateUpdate::nonce(a, 7), + StateUpdate::Account { + address: a, + patch: AccountPatch::default().nonce(7), + } + ); + assert_eq!( + StateUpdate::code(a, Bytes::from_static(&[0x60])), + StateUpdate::Account { + address: a, + patch: AccountPatch::default().code(Bytes::from_static(&[0x60])), + } + ); + let patch = AccountPatch::default().balance(U256::from(1)).nonce(2); + assert_eq!( + StateUpdate::account(a, patch.clone()), + StateUpdate::Account { address: a, patch } + ); +} + +// =========================================================================== +// §16.8 — Decision-2 write-through pins for the remaining protocols injectors. +// =========================================================================== + +#[cfg(feature = "protocols")] +#[tokio::test] +async fn inject_v2_pool_metadata_writes_through_to_backend() -> Result<()> { + use evm_fork_cache::cache::V2PoolMetadata; + + let pool = Address::repeat_byte(0xb3); + let mut cache = setup_cache().await?; + let meta = V2PoolMetadata { + token0: Address::repeat_byte(0x01), + token1: Address::repeat_byte(0x02), + last_block_timestamp: 0, + }; + + cache.inject_v2_pool_metadata(pool, &meta)?; + + assert!( + cache.pool_storage_slot_count(pool) > 0, + "inject_v2_pool_metadata must write through to the backend (Decision 2)" + ); + Ok(()) +} + +#[cfg(feature = "protocols")] +#[tokio::test] +async fn inject_v3_ticks_writes_through_to_backend() -> Result<()> { + use evm_fork_cache::cache::TickInfo; + use std::collections::HashMap; + + let pool = Address::repeat_byte(0xb4); + let mut cache = setup_cache().await?; + let mut ticks = HashMap::new(); + ticks.insert( + 0i32, + TickInfo { + liquidity_gross: 100, + liquidity_net: 50, + initialized: true, + }, + ); + + let injected = cache.inject_v3_ticks(pool, &ticks)?; + assert!(injected > 0); + assert!( + cache.pool_storage_slot_count(pool) > 0, + "inject_v3_ticks must write through to the backend (Decision 2)" + ); + Ok(()) +} From 55acddf58bc64dbbfdc8d5ef43d32633af478201 Mon Sep 17 00:00:00 2001 From: Kai Aldag Date: Tue, 16 Jun 2026 10:24:07 +0100 Subject: [PATCH 12/26] Phase 3 fix-review: extend account_state-awareness to the snapshot + account-info paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An adversarial review of the §16 fixes found the §16.0 `cached_storage_value` fix had not been propagated to two sibling read paths, leaving the same silent-corruption class open elsewhere. This completes it. create_snapshot (HIGH — was: snapshot/overlay/validator read a shadowed slot where the live cache now reads ZERO): - `EvmSnapshot` gains a `storage_cleared: HashSet
` set. `create_snapshot` captures a `StorageCleared`/`NotExisting` overlay account's storage as ONLY its overlay slots (shadowed backend slots dropped) and records the address in the set. - `EvmSnapshot::storage_value` and `EvmOverlay::storage` honor the set: a slot absent from a cleared account reads ZERO and does NOT fall through to the backend / `ext_db` — mirroring `cached_storage_value` and the live EVM SLOAD. This also realigns the background validator's `old` (snapshot value) with the synchronous `verify_slots` (`cached_storage_value`), which had diverged. loaded_account_info (MED — account-axis analog): - Mirror revm `DbAccount::info()`: a `NotExisting` overlay account is absent to the EVM (returns `None`) and does not fall through to the backend, so a `BalanceDelta` / partial `Account` patch skips rather than computing against a stale `info` the EVM never sees. write_account_info_through (LOW — layer hygiene): - Normalize a `ZERO` `code_hash` to `KECCAK_EMPTY` before the backend write so both layers store an identical hash (matching revm's `insert_contract`, which the overlay write already applies). Tests: - New regressions: `snapshot_mirrors_live_read_for_cleared_account` (snapshot/overlay read ZERO for a cleared account's shadowed slot), `balance_delta_on_notexisting_overlay_account_is_skipped`, `account_patch_normalizes_zero_code_hash_across_layers`. - Freshness seed corrections: the optimistic-loop tests (incl. the `cache_with_balance` helper) seeded balances backend-only on the StorageCleared MockERC20 fixture — invisible to the EVM/snapshot post-fix. Reseeded overlay-resident (EVM-visible) so they exercise a realistic, EVM-consistent state; assertions unchanged. These tests had been passing only because the pre-fix snapshot read the shadowed value (the bug this commit closes). Full suite green (253 tests + 31 doctests), clippy default + --no-default-features, fmt, RUSTDOCFLAGS=-D warnings doc, and cargo bench --no-run. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 16 ++++++++ src/cache/mod.rs | 50 +++++++++++++++++++----- src/cache/overlay.rs | 12 ++++++ src/cache/snapshot.rs | 29 +++++++++++--- tests/freshness.rs | 43 ++++++++++++++++----- tests/snapshot_overlay.rs | 37 ++++++++++++++++++ tests/state_update.rs | 80 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 242 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fe44fa..3187daf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -140,6 +140,22 @@ pre-release development phases (see [`docs/ROADMAP.md`](docs/ROADMAP.md)). changes, instead of unconditionally inserting `AccountInfo::default()` into the shared backend for an all-`None` (or value-unchanged) patch on an absent address. A real field change still materializes the backend account (unchanged intent). +- **`account_state`-awareness extended to the snapshot + account-info paths** + (Phase 3 fix-review, HIGH + MED). A follow-up adversarial review found the §16.0 + `cached_storage_value` fix had not been propagated to two sibling read paths: + - `create_snapshot` now mirrors the live read: a `StorageCleared`/`NotExisting` + account's storage is captured as **only** its overlay slots (shadowed backend + slots dropped) and recorded in a new `EvmSnapshot.storage_cleared` set, so + `EvmSnapshot::storage_value` and snapshot-backed `EvmOverlay`s read such a + slot as ZERO instead of the shadowed backend value (which also kept the + background freshness validator's `old` consistent with `verify_slots`). The + `EvmOverlay` storage read honors the set and does **not** fall through to its + `ext_db` for a cleared account. + - `loaded_account_info` now mirrors revm `DbAccount::info()`: a `NotExisting` + overlay account is treated as absent (returns `None`), so a `BalanceDelta` / + partial `Account` patch skips rather than computing against a stale `info`. + - `write_account_info_through` normalizes a `ZERO` `code_hash` to `KECCAK_EMPTY` + so both cache layers store an identical hash (matching revm's `insert_contract`). ### Notes diff --git a/src/cache/mod.rs b/src/cache/mod.rs index bb94329..ebe2897 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -1585,18 +1585,30 @@ impl EvmCache { /// backend), without touching RPC. `None` when the account is absent from /// both layers. fn loaded_account_info(&self, address: Address) -> Option { - self.db - .cache - .accounts - .get(&address) - .map(|a| a.info.clone()) - .or_else(|| self.blockchain_db.accounts().read().get(&address).cloned()) + if let Some(a) = self.db.cache.accounts.get(&address) { + // Mirror revm `DbAccount::info()` / `basic_ref`: a NotExisting overlay + // account is absent to the EVM (returns None) and does NOT fall through + // to the backend. Without this, a relative balance update / partial + // patch would compute against a stale `info` the EVM never sees. + if matches!(a.account_state, AccountState::NotExisting) { + return None; + } + return Some(a.info.clone()); + } + self.blockchain_db.accounts().read().get(&address).cloned() } /// Write an `AccountInfo` through both layers, mirroring the slot policy: /// backend (layer 2) always; overlay (layer 1) only if an overlay account /// already exists (never materialize a new overlay account). - fn write_account_info_through(&mut self, address: Address, info: AccountInfo) { + fn write_account_info_through(&mut self, address: Address, mut info: AccountInfo) { + // Normalize the code hash the way revm's `insert_contract` (applied on the + // overlay write below) does, so both layers store an identical hash: a ZERO + // code_hash denotes empty code → KECCAK_EMPTY. Otherwise the overlay would + // hold KECCAK_EMPTY while the backend kept ZERO for the same account. + if info.code_hash == B256::ZERO { + info.code_hash = revm::primitives::KECCAK_EMPTY; + } let overlay_present = self.db.cache.accounts.contains_key(&address); { let mut accounts = self.blockchain_db.accounts().write(); @@ -1929,20 +1941,38 @@ impl EvmCache { } // 2. Overlay from CacheDB (Layer 1, takes precedence) + let mut storage_cleared = std::collections::HashSet::new(); for (addr, db_account) in &self.db.cache.accounts { if let Some(code) = &db_account.info.code { code_by_hash.insert(db_account.info.code_hash, code.clone()); } accounts.insert(*addr, db_account.info.clone()); - let account_storage = storage.entry(*addr).or_default(); - for (slot, value) in &db_account.storage { - account_storage.insert(*slot, *value); + + // Mirror the live read path (cached_storage_value / the EVM SLOAD): a + // StorageCleared/NotExisting account's storage is locally complete, so + // the snapshot holds ONLY its overlay slots (any shadowed backend slots + // are dropped) and an absent slot reads ZERO via `storage_cleared`, + // rather than falling through to the (shadowed) backend or an ext_db. + if matches!( + db_account.account_state, + AccountState::StorageCleared | AccountState::NotExisting + ) { + storage_cleared.insert(*addr); + let account_storage: HashMap = + db_account.storage.iter().map(|(k, v)| (*k, *v)).collect(); + storage.insert(*addr, account_storage); + } else { + let account_storage = storage.entry(*addr).or_default(); + for (slot, value) in &db_account.storage { + account_storage.insert(*slot, *value); + } } } Arc::new(snapshot::EvmSnapshot { accounts, storage, + storage_cleared, block_hashes: HashMap::new(), code_by_hash, block_number: self.block_number, diff --git a/src/cache/overlay.rs b/src/cache/overlay.rs index 4831517..4b97288 100644 --- a/src/cache/overlay.rs +++ b/src/cache/overlay.rs @@ -592,6 +592,12 @@ impl Database for EvmOverlay { { return Ok(*value); } + // 2b. A cleared account's storage is locally complete: an absent slot reads + // ZERO and must NOT fall through to the ext_db, mirroring the live EVM + // SLOAD for a StorageCleared/NotExisting account. + if self.snapshot.storage_cleared.contains(&address) { + return Ok(U256::ZERO); + } // 3. RPC fallback if let Some(ref ext_db) = self.ext_db { let value = ext_db.storage_ref(address, index)?; @@ -659,6 +665,7 @@ mod tests { accounts, storage: HashMap::new(), block_hashes: HashMap::new(), + storage_cleared: std::collections::HashSet::new(), code_by_hash: HashMap::new(), block_number: None, basefee: None, @@ -691,6 +698,7 @@ mod tests { accounts: HashMap::new(), storage, block_hashes: HashMap::new(), + storage_cleared: std::collections::HashSet::new(), code_by_hash: HashMap::new(), block_number: None, basefee: None, @@ -721,6 +729,7 @@ mod tests { accounts: HashMap::new(), storage, block_hashes: HashMap::new(), + storage_cleared: std::collections::HashSet::new(), code_by_hash: HashMap::new(), block_number: None, basefee: None, @@ -752,6 +761,7 @@ mod tests { accounts: HashMap::new(), storage: HashMap::new(), block_hashes: HashMap::new(), + storage_cleared: std::collections::HashSet::new(), code_by_hash: HashMap::new(), block_number: None, basefee: None, @@ -784,6 +794,7 @@ mod tests { accounts: HashMap::new(), storage: HashMap::new(), block_hashes: HashMap::new(), + storage_cleared: std::collections::HashSet::new(), code_by_hash, block_number: None, basefee: None, @@ -809,6 +820,7 @@ mod tests { let snapshot = Arc::new(EvmSnapshot { accounts: HashMap::new(), storage: HashMap::new(), + storage_cleared: std::collections::HashSet::new(), block_hashes, code_by_hash: HashMap::new(), block_number: None, diff --git a/src/cache/snapshot.rs b/src/cache/snapshot.rs index d36364b..3019ec6 100644 --- a/src/cache/snapshot.rs +++ b/src/cache/snapshot.rs @@ -28,7 +28,7 @@ //! //! [`EvmOverlay`]: super::EvmOverlay -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use alloy_primitives::{Address, B256, U256}; use revm::primitives::hardfork::SpecId; @@ -45,6 +45,11 @@ use revm::state::{AccountInfo, Bytecode}; pub struct EvmSnapshot { pub(crate) accounts: HashMap, pub(crate) storage: HashMap>, + /// Accounts whose storage is locally complete (revm `StorageCleared` / + /// `NotExisting`): a slot absent from `storage` for such an account reads as + /// ZERO and must NOT fall through to an `ext_db`, mirroring the live EVM SLOAD + /// and [`EvmCache::cached_storage_value`](super::EvmCache::cached_storage_value). + pub(crate) storage_cleared: HashSet
, pub(crate) block_hashes: HashMap, /// Bytecode lookup by code_hash (derived from accounts at creation time). pub(crate) code_by_hash: HashMap, @@ -60,15 +65,26 @@ pub struct EvmSnapshot { } impl EvmSnapshot { - /// Return the snapshot's value for a storage slot, if present. + /// Return the snapshot's value for a storage slot, mirroring the live read. /// - /// Used by the freshness validator to compare a freshly-fetched value - /// against the value the snapshot was built from. A missing entry means the - /// snapshot never captured that slot (it would read as zero in a sim). + /// Used by the freshness validator to compare a freshly-fetched value against + /// the value the snapshot was built from. Resolution matches + /// [`EvmCache::cached_storage_value`](super::EvmCache::cached_storage_value): + /// a captured slot returns its value; a slot absent from a cleared account + /// (revm `StorageCleared`/`NotExisting`) returns `Some(ZERO)` (its storage is + /// locally complete); any other absent slot returns `None`. pub fn storage_value(&self, address: Address, slot: U256) -> Option { - self.storage + if let Some(value) = self + .storage .get(&address) .and_then(|s| s.get(&slot).copied()) + { + return Some(value); + } + if self.storage_cleared.contains(&address) { + return Some(U256::ZERO); + } + None } } @@ -89,6 +105,7 @@ mod tests { let snap = EvmSnapshot { accounts: HashMap::new(), storage: HashMap::new(), + storage_cleared: HashSet::new(), block_hashes: HashMap::new(), code_by_hash: HashMap::new(), block_number: Some(100), diff --git a/tests/freshness.rs b/tests/freshness.rs index 1c981a5..9cc8c21 100644 --- a/tests/freshness.rs +++ b/tests/freshness.rs @@ -335,7 +335,13 @@ async fn cache_with_balance(token: Address, owner: Address, balance: U256) -> Re install_default_account(&mut cache, owner); install_mock_erc20(&mut cache, token); if balance > U256::ZERO { - cache.inject_storage_batch(&[(token, balance_slot_for(owner), balance)]); + // Overlay-resident seed so the balance is EVM-visible: `token` is a + // StorageCleared MockERC20, so a backend-only seed reads as ZERO via the + // account_state-aware read path (invisible to the optimistic sim and the + // snapshot). Mirrors `state_update::balance_tracking_scenario`. + cache + .db_mut() + .insert_account_storage(token, balance_slot_for(owner), balance)?; } Ok(cache) } @@ -390,10 +396,14 @@ async fn run_mismatch_path_corrected_only_affected_rerun() -> Result<()> { install_default_account(&mut cache, owner2); install_mock_erc20(&mut cache, token); install_mock_erc20(&mut cache, token2); - cache.inject_storage_batch(&[ - (token, balance_slot_for(owner), U256::from(1000)), - (token2, balance_slot_for(owner2), U256::from(5000)), - ]); + // Overlay-resident seeds (EVM-visible): both tokens are StorageCleared, so a + // backend-only seed would read ZERO via the account_state-aware read path. + cache + .db_mut() + .insert_account_storage(token, balance_slot_for(owner), U256::from(1000))?; + cache + .db_mut() + .insert_account_storage(token2, balance_slot_for(owner2), U256::from(5000))?; // Fetcher: owner's balance slot DROPPED to 50 (< the 100 transfer, so the // re-run now reverts); owner2's slot unchanged; recipient slots read as zero @@ -767,7 +777,11 @@ async fn optimistic_result_reports_status_per_outcome() -> Result<()> { install_default_account(&mut cache, owner); install_mock_erc20(&mut cache, token); let slot = balance_slot_for(owner); - cache.inject_storage_batch(&[(token, slot, U256::from(1000))]); + // Overlay-resident (EVM-visible) seed: `token` is a StorageCleared MockERC20, + // so a backend-only seed reads ZERO via the account_state-aware read path. + cache + .db_mut() + .insert_account_storage(token, slot, U256::from(1000))?; cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([( (token, slot), U256::from(1000), @@ -823,7 +837,11 @@ async fn dropping_after_fetch_started_suppresses_correction() -> Result<()> { install_default_account(&mut cache, owner); install_mock_erc20(&mut cache, token); let slot = balance_slot_for(owner); - cache.inject_storage_batch(&[(token, slot, U256::from(1000))]); + // Overlay-resident (EVM-visible) seed: `token` is a StorageCleared MockERC20, + // so a backend-only seed reads ZERO via the account_state-aware read path. + cache + .db_mut() + .insert_account_storage(token, slot, U256::from(1000))?; // Two rendezvous: R1 = "fetch started", R2 = "released by the test". After R2 // the fetcher reports a CHANGED balance, so absent the cancel the validator @@ -903,7 +921,10 @@ async fn run_corrected_rerun_verifies_newly_read_volatile_slot() -> Result<()> { let slot_a = U256::from(0); let slot_b = U256::from(1); // Snapshot: A = 5 (nonzero) → optimistic takes "return A" and never reads B. - cache.inject_storage_batch(&[(contract, slot_a, U256::from(5))]); + // Overlay-resident (EVM-visible) seed: `contract` is StorageCleared. + cache + .db_mut() + .insert_account_storage(contract, slot_a, U256::from(5))?; // Fresh chain: A dropped to 0 (flips the branch) and B is 777. cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([ ((contract, slot_a), U256::from(0)), @@ -1017,7 +1038,11 @@ async fn run_honors_tx_gas_limit() -> Result<()> { install_default_account(&mut cache, owner); install_mock_erc20(&mut cache, token); let slot = balance_slot_for(owner); - cache.inject_storage_batch(&[(token, slot, U256::from(1000))]); + // Overlay-resident (EVM-visible) seed: `token` is a StorageCleared MockERC20, + // so a backend-only seed reads ZERO via the account_state-aware read path. + cache + .db_mut() + .insert_account_storage(token, slot, U256::from(1000))?; cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([( (token, slot), U256::from(1000), diff --git a/tests/snapshot_overlay.rs b/tests/snapshot_overlay.rs index 23a204d..cc3a510 100644 --- a/tests/snapshot_overlay.rs +++ b/tests/snapshot_overlay.rs @@ -177,3 +177,40 @@ async fn overlay_reads_reflect_snapshot_state() -> Result<()> { Ok(()) } + +/// Regression (§16 fix-review HIGH): `create_snapshot` must mirror the live +/// account-state-aware read. A `StorageCleared` account with a backend-only +/// (shadowed) slot reads ZERO live; the snapshot, `storage_value`, and a +/// snapshot-backed overlay must all agree — not the shadowed backend value. Pre- +/// fix the snapshot/overlay read the shadowed 100 while the live cache read 0. +#[tokio::test] +async fn snapshot_mirrors_live_read_for_cleared_account() -> Result<()> { + let token = Address::repeat_byte(0x5c); + let slot = U256::from(MOCK_ERC20_BALANCE_SLOT); // absent from the cleared overlay + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, token); // sets account_state = StorageCleared + cache.inject_storage_batch(&[(token, slot, U256::from(100))]); // backend-only shadow + + // Live read is ZERO (the §16.0 fix). + assert_eq!(cache.cached_storage_value(token, slot), Some(U256::ZERO)); + + let snapshot: Arc = cache.create_snapshot(); + assert_eq!( + snapshot.storage_value(token, slot), + Some(U256::ZERO), + "snapshot.storage_value must mirror the live cleared read, not the shadowed 100" + ); + + // A snapshot-backed overlay (no ext_db, as the freshness validator uses) must + // also read ZERO for the cleared account's absent slot. + let mut overlay = EvmOverlay::new(Arc::clone(&snapshot), None); + let value = overlay + .storage(token, slot) + .map_err(|e| anyhow!("overlay storage read failed: {e:?}"))?; + assert_eq!( + value, + U256::ZERO, + "snapshot-backed overlay must read ZERO for a cleared account's absent slot" + ); + Ok(()) +} diff --git a/tests/state_update.rs b/tests/state_update.rs index 5ff4225..825cc31 100644 --- a/tests/state_update.rs +++ b/tests/state_update.rs @@ -1526,3 +1526,83 @@ async fn inject_v3_ticks_writes_through_to_backend() -> Result<()> { ); Ok(()) } + +// =========================================================================== +// §16 fix-review regressions — account_state-awareness on the account axis. +// =========================================================================== + +#[tokio::test] +async fn balance_delta_on_notexisting_overlay_account_is_skipped() -> Result<()> { + // A NotExisting overlay account is absent to the EVM (revm DbAccount::info() + // returns None), even if it carries a stale info.balance. loaded_account_info + // must treat it as cold so a BalanceDelta skips rather than applying to 1000. + use revm::database::AccountState; + let acct = Address::repeat_byte(0x7e); + let mut cache = setup_cache().await?; + cache.db_mut().insert_account_info( + acct, + AccountInfo { + balance: U256::from(1000), + ..Default::default() + }, + ); + cache + .db_mut() + .cache + .accounts + .get_mut(&acct) + .expect("overlay account present") + .account_state = AccountState::NotExisting; + + let diff = cache.apply_update(&StateUpdate::balance_delta( + acct, + SlotDelta::Add(U256::from(500)), + )); + + assert!( + diff.accounts.is_empty(), + "NotExisting account is EVM-absent: the delta must skip, not apply to the stale 1000" + ); + assert_eq!( + diff.skipped_balances, + vec![SkippedBalanceDelta { + address: acct, + delta: SlotDelta::Add(U256::from(500)), + }] + ); + Ok(()) +} + +#[tokio::test] +async fn account_patch_normalizes_zero_code_hash_across_layers() -> Result<()> { + // write_account_info_through normalizes a ZERO code_hash to KECCAK_EMPTY so both + // layers agree (the overlay write does this via insert_contract; the backend + // write must too). Seed the backend (unnormalized) with a ZERO hash, then patch. + use alloy_primitives::B256; + use revm::primitives::KECCAK_EMPTY; + let acct = Address::repeat_byte(0x7f); + let mut cache = setup_cache().await?; + cache.blockchain_db().accounts().write().insert( + acct, + AccountInfo { + balance: U256::from(1), + code_hash: B256::ZERO, + ..Default::default() + }, + ); + + cache.apply_update(&StateUpdate::balance(acct, U256::from(2))); + + let backend_hash = cache + .blockchain_db() + .accounts() + .read() + .get(&acct) + .map(|i| i.code_hash); + assert_eq!( + backend_hash, + Some(KECCAK_EMPTY), + "backend code_hash must be normalized to KECCAK_EMPTY, not left ZERO" + ); + Ok(()) +} From d027bb387a665be470f366de5e5a9bcbb32a0c2a Mon Sep 17 00:00:00 2001 From: Kai Aldag Date: Tue, 16 Jun 2026 11:39:47 +0100 Subject: [PATCH 13/26] Phase 2 review + round-2 fixes: validator trust contract + account-axis account_state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses a fresh Phase 2 freshness review (4 findings) and the round-2 adversarial pass on the snapshot fixes (1 HIGH + LOWs). Unifying theme: the validator must never return a trusted verdict on incomplete verification, and account_state- awareness must hold on EVERY read/flatten path (storage AND account/basic). Validator trust contract (src/freshness.rs): - Fixed-point round cap now returns `Validation::Unverified` (was a best-effort, trusted `Corrected`) and queues no corrections — the results had not reached a verified fixed point. (P1) - A corrected re-run that fails to execute (host/transact `Err`, not a revert/halt) returns `Unverified` instead of silently keeping the stale optimistic result. (P2) - New `collect_fetch_results` requires the batch fetcher to return EVERY requested slot; an omitted slot → `Unverified` rather than defaulting to zero (a custom fetcher dropping a slot could otherwise cause a false confirm/correct). (P2) Account-axis account_state (round-2 HIGH + LOWs): - `create_snapshot` / `EvmOverlay::basic`: a `NotExisting` overlay account is now excluded from the snapshot `accounts`/`code_by_hash` and recorded in a new `EvmSnapshot.accounts_not_existing` set; `basic` returns `None` for it (no ext_db fall-through), mirroring revm `DbAccount::info()` and `loaded_account_info`. Pre-fix the snapshot/parallel/validator path saw a phantom existing account. - `target_account_info` (deploy path): `NotExisting` overlay account treated as a missing target rather than returning stale info. - `loaded_account_info`: normalizes a `ZERO` code_hash to `KECCAK_EMPTY` at load, so a patch's `old_code_hash` matches what is written (self-consistent diff). - `modify_account_balance` doc corrected (a `NotExisting` account is also cold). Access-list checkpoint (src/cache/overlay.rs + mod.rs): - `call_raw_with_access_list` / `call_raw_with_access_list_with` now `checkpoint_revert` on every path (success and host error) via a `match` instead of `?`-before-revert. Resolves KNOWN_ISSUES #9. Tests (regressions, red before / green after): - `snapshot_basic_returns_none_for_notexisting_account` (account-axis HIGH). - `run_unverified_when_fixed_point_round_cap_exceeded` (P1; drives a generated 12-deep chained-SLOAD contract past the 8-round cap, asserts Unverified + pending_len()==0). - `run_unverified_when_fetcher_omits_requested_slot` (P2; omitting fetcher). Not separately unit-tested (covered by the code change; not cleanly triggerable offline): the corrected-rerun host-error path and the deploy-path `target_account_info` guard. Docs: PurgeScope StorageCleared/NotExisting refetch caveat; KNOWN_ISSUES #9 marked resolved; CHANGELOG. Full suite green (256 tests + 31 doctests), clippy default + --no-default-features, fmt, RUSTDOCFLAGS=-D warnings doc, cargo bench --no-run. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 28 ++++++++ docs/KNOWN_ISSUES.md | 11 ++-- src/cache/mod.rs | 105 ++++++++++++++++++++---------- src/cache/overlay.rs | 43 +++++++++---- src/cache/snapshot.rs | 6 ++ src/freshness.rs | 131 ++++++++++++++++++++++++-------------- src/state_update.rs | 11 ++++ tests/freshness.rs | 119 ++++++++++++++++++++++++++++++++++ tests/snapshot_overlay.rs | 44 +++++++++++++ 9 files changed, 398 insertions(+), 100 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3187daf..bb58ca9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -156,6 +156,34 @@ pre-release development phases (see [`docs/ROADMAP.md`](docs/ROADMAP.md)). partial `Account` patch skips rather than computing against a stale `info`. - `write_account_info_through` normalizes a `ZERO` `code_hash` to `KECCAK_EMPTY` so both cache layers store an identical hash (matching revm's `insert_contract`). +- **`account_state`-awareness completed on the account (`basic`) axis** (round-2 + review, HIGH). The snapshot path still leaked a `NotExisting` account's stale + info: `create_snapshot` inserted it into `accounts`, so `EvmOverlay::basic` + returned a phantom existing account where live revm / `loaded_account_info` + return `None`. Now `create_snapshot` excludes `NotExisting` accounts from + `accounts`/`code_by_hash` and records them in a new + `EvmSnapshot.accounts_not_existing` set; `EvmOverlay::basic` returns `None` for + them (no `ext_db` fall-through). `target_account_info` (deploy path) and + `loaded_account_info` (code_hash normalized at load) were brought into line too. +- **Freshness validator trust contract hardened** (Phase 2 review). The + background validator no longer returns a *trusted* verdict on incomplete or + ambiguous verification: + - **Fixed-point round cap → `Unverified`.** Exceeding `MAX_VALIDATION_ROUNDS` + (corrections kept opening new volatile slots) now returns + `Validation::Unverified` and queues no corrections, instead of a best-effort + `Corrected` resting on un-verified state. + - **Corrected re-run host error → `Unverified`.** A failed corrected re-run + (a `transact` error, not a revert/halt) returns `Unverified` rather than + silently keeping the stale optimistic result. + - **Missing fetcher results → `Unverified`.** A new `collect_fetch_results` + helper requires the batch fetcher to return *every* requested slot; an omitted + slot yields `Unverified` instead of defaulting to zero (which could produce a + false confirmation/correction with a custom fetcher). +- **`call_raw_with_access_list*` reverts its checkpoint on transact errors** + (Phase 2 review; `EvmCache` + `EvmOverlay`). Previously the host-error path + `?`-returned before `checkpoint_revert`, leaving the journal checkpoint + un-reverted; both methods now revert on every path. (See `docs/KNOWN_ISSUES.md` + #9, now resolved.) ### Notes diff --git a/docs/KNOWN_ISSUES.md b/docs/KNOWN_ISSUES.md index ac00fb9..06a755d 100644 --- a/docs/KNOWN_ISSUES.md +++ b/docs/KNOWN_ISSUES.md @@ -59,11 +59,12 @@ Confidence legend: **[V]** verified against the source during review; Either the field is meaningless on this path or the population was missed — the docs now state the field is empty here; reconcile before relying on it. -9. **[V] `call_raw_with_access_list` does not revert its checkpoint on a transact - error.** It propagates the EVM `transact` error with `?` *before* reverting the - journaled checkpoint, whereas `call_raw` / `simulate_with_transfer_tracking` - revert on every path. A host-level transact error therefore leaves the overlay - checkpoint un-reverted. (Reverts normally on success and on revert/halt.) +9. **[FIXED] `call_raw_with_access_list` did not revert its checkpoint on a + transact error.** Both `EvmCache::call_raw_with_access_list` and + `EvmOverlay::call_raw_with_access_list_with` now match on the `transact_one` + result and `checkpoint_revert` on **every** path (success and host error), + matching `call_raw` / `simulate_with_transfer_tracking`. A host-level transact + error no longer leaves the overlay checkpoint un-reverted. 10. **[V] `SystemTime::now().unwrap()` panic risk in EVM construction.** `build_evm` / `make_local_context` (and the overlay equivalents) call diff --git a/src/cache/mod.rs b/src/cache/mod.rs index ebe2897..1b4a458 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -1510,9 +1510,10 @@ impl EvmCache { /// balance changed; /// - `None` writes nothing (no account is materialized) and returns `None`. /// - /// "Cold" for a balance is the account being absent from both layers; the - /// revm `account_state` does **not** matter here (it governs storage, not the - /// basic `AccountInfo`). To skip cold accounts, map through the `Option`: + /// "Cold" for a balance is the account being absent from both layers — or + /// present in the overlay as revm `NotExisting` (absent to the EVM), which the + /// internal account read also treats as cold, mirroring `DbAccount::info()`. + /// To skip cold accounts, map through the `Option`: /// `|cur| cur.map(|v| v.saturating_add(amount))`. /// /// ```no_run @@ -1585,7 +1586,7 @@ impl EvmCache { /// backend), without touching RPC. `None` when the account is absent from /// both layers. fn loaded_account_info(&self, address: Address) -> Option { - if let Some(a) = self.db.cache.accounts.get(&address) { + let mut info = if let Some(a) = self.db.cache.accounts.get(&address) { // Mirror revm `DbAccount::info()` / `basic_ref`: a NotExisting overlay // account is absent to the EVM (returns None) and does NOT fall through // to the backend. Without this, a relative balance update / partial @@ -1593,9 +1594,22 @@ impl EvmCache { if matches!(a.account_state, AccountState::NotExisting) { return None; } - return Some(a.info.clone()); + a.info.clone() + } else { + self.blockchain_db + .accounts() + .read() + .get(&address) + .cloned()? + }; + // Normalize like revm `insert_contract`: a ZERO code_hash denotes empty + // code -> KECCAK_EMPTY. Done at load time so a patch's `old_code_hash` + // matches what `write_account_info_through` stores (a self-consistent diff, + // no phantom/under-reported code_hash change). + if info.code_hash == B256::ZERO { + info.code_hash = revm::primitives::KECCAK_EMPTY; } - self.blockchain_db.accounts().read().get(&address).cloned() + Some(info) } /// Write an `AccountInfo` through both layers, mirroring the slot policy: @@ -1942,21 +1956,33 @@ impl EvmCache { // 2. Overlay from CacheDB (Layer 1, takes precedence) let mut storage_cleared = std::collections::HashSet::new(); + let mut accounts_not_existing = std::collections::HashSet::new(); for (addr, db_account) in &self.db.cache.accounts { - if let Some(code) = &db_account.info.code { - code_by_hash.insert(db_account.info.code_hash, code.clone()); + let not_existing = matches!(db_account.account_state, AccountState::NotExisting); + let cleared = + not_existing || matches!(db_account.account_state, AccountState::StorageCleared); + + // Account info. Mirror revm `DbAccount::info()` / `loaded_account_info`: + // a NotExisting overlay account is absent to the EVM (`basic` returns + // None), so it must NOT contribute info/code to the snapshot — and any + // backend-merged entry from step 1 is dropped, since loaded_account_info + // short-circuits to None before consulting the backend. + if not_existing { + accounts_not_existing.insert(*addr); + accounts.remove(addr); + } else { + if let Some(code) = &db_account.info.code { + code_by_hash.insert(db_account.info.code_hash, code.clone()); + } + accounts.insert(*addr, db_account.info.clone()); } - accounts.insert(*addr, db_account.info.clone()); - // Mirror the live read path (cached_storage_value / the EVM SLOAD): a - // StorageCleared/NotExisting account's storage is locally complete, so - // the snapshot holds ONLY its overlay slots (any shadowed backend slots - // are dropped) and an absent slot reads ZERO via `storage_cleared`, - // rather than falling through to the (shadowed) backend or an ext_db. - if matches!( - db_account.account_state, - AccountState::StorageCleared | AccountState::NotExisting - ) { + // Storage. A StorageCleared/NotExisting account's storage is locally + // complete: the snapshot holds ONLY its overlay slots (any shadowed + // backend slots are dropped) and an absent slot reads ZERO via + // `storage_cleared`, rather than falling through to the (shadowed) + // backend or an ext_db. + if cleared { storage_cleared.insert(*addr); let account_storage: HashMap = db_account.storage.iter().map(|(k, v)| (*k, *v)).collect(); @@ -1973,6 +1999,7 @@ impl EvmCache { accounts, storage, storage_cleared, + accounts_not_existing, block_hashes: HashMap::new(), code_by_hash, block_number: self.block_number, @@ -2558,24 +2585,29 @@ impl EvmCache { let mut evm = self.build_evm(); let checkpoint = evm.journaled_state.checkpoint(); - let result = evm - .transact_one(tx) - .map_err(|e| anyhow!("Failed to transact: {:?}", e))?; - - // Extract access list from journaled state before reverting. - // After transact_one, journaled_state.state contains all touched accounts/slots. - let mut access_list = StorageAccessList::default(); - for (address, account) in evm.journaled_state.state.iter() { - if account.is_touched() { - access_list.accounts.insert(*address); - for (slot_key, _) in account.storage.iter() { - access_list.slots.insert((*address, *slot_key)); + match evm.transact_one(tx) { + Ok(result) => { + // Extract access list from journaled state before reverting. After + // transact_one, journaled_state.state holds all touched accounts/slots. + let mut access_list = StorageAccessList::default(); + for (address, account) in evm.journaled_state.state.iter() { + if account.is_touched() { + access_list.accounts.insert(*address); + for (slot_key, _) in account.storage.iter() { + access_list.slots.insert((*address, *slot_key)); + } + } } + evm.journaled_state.checkpoint_revert(checkpoint); + Ok((result, access_list)) + } + Err(e) => { + // Revert the checkpoint even on a host/transact error so the EVM + // journal is not left dirty (mirrors `call_raw`). + evm.journaled_state.checkpoint_revert(checkpoint); + Err(anyhow!("Failed to transact: {:?}", e)) } } - - evm.journaled_state.checkpoint_revert(checkpoint); - Ok((result, access_list)) } /// Execute a call and return its emitted logs and gas used. @@ -3399,7 +3431,12 @@ impl EvmCache { missing_target: MissingTargetBehavior, ) -> Result { if let Some(account) = self.db.cache.accounts.get(&target) { - return Ok(account.info.clone()); + // A NotExisting overlay account is absent to the EVM (revm + // `DbAccount::info()` returns None); treat it as a missing target + // rather than returning its stale/default info. + if !matches!(account.account_state, AccountState::NotExisting) { + return Ok(account.info.clone()); + } } match missing_target { diff --git a/src/cache/overlay.rs b/src/cache/overlay.rs index 4b97288..7656e22 100644 --- a/src/cache/overlay.rs +++ b/src/cache/overlay.rs @@ -467,22 +467,27 @@ impl EvmOverlay { let mut evm = self.build_evm(); use revm::context_interface::JournalTr; let checkpoint = evm.journaled_state.checkpoint(); - let result = evm - .transact_one(tx_env) - .map_err(|e| anyhow!("Failed to transact: {:?}", e))?; - - let mut access_list = StorageAccessList::default(); - for (address, account) in evm.journaled_state.state.iter() { - if account.is_touched() { - access_list.accounts.insert(*address); - for (slot_key, _) in account.storage.iter() { - access_list.slots.insert((*address, *slot_key)); + match evm.transact_one(tx_env) { + Ok(result) => { + let mut access_list = StorageAccessList::default(); + for (address, account) in evm.journaled_state.state.iter() { + if account.is_touched() { + access_list.accounts.insert(*address); + for (slot_key, _) in account.storage.iter() { + access_list.slots.insert((*address, *slot_key)); + } + } } + evm.journaled_state.checkpoint_revert(checkpoint); + Ok((result, access_list)) + } + Err(e) => { + // Revert the checkpoint even on a host/transact error so the EVM + // journal is not left dirty (mirrors `call_raw`). + evm.journaled_state.checkpoint_revert(checkpoint); + Err(anyhow!("Failed to transact: {:?}", e)) } } - - evm.journaled_state.checkpoint_revert(checkpoint); - Ok((result, access_list)) } /// Write a storage value into this overlay's dirty layer. @@ -548,6 +553,12 @@ impl Database for EvmOverlay { if let Some(info) = self.snapshot.accounts.get(&address) { return Ok(Some(info.clone())); } + // 2b. A NotExisting account is absent to the EVM: return None and do NOT + // fall through to the ext_db, mirroring revm `DbAccount::info()` and the + // live `EvmCache` account read (symmetric with `storage_cleared`). + if self.snapshot.accounts_not_existing.contains(&address) { + return Ok(None); + } // 3. RPC fallback if let Some(ref ext_db) = self.ext_db { let info = ext_db.basic_ref(address)?; @@ -666,6 +677,7 @@ mod tests { storage: HashMap::new(), block_hashes: HashMap::new(), storage_cleared: std::collections::HashSet::new(), + accounts_not_existing: std::collections::HashSet::new(), code_by_hash: HashMap::new(), block_number: None, basefee: None, @@ -699,6 +711,7 @@ mod tests { storage, block_hashes: HashMap::new(), storage_cleared: std::collections::HashSet::new(), + accounts_not_existing: std::collections::HashSet::new(), code_by_hash: HashMap::new(), block_number: None, basefee: None, @@ -730,6 +743,7 @@ mod tests { storage, block_hashes: HashMap::new(), storage_cleared: std::collections::HashSet::new(), + accounts_not_existing: std::collections::HashSet::new(), code_by_hash: HashMap::new(), block_number: None, basefee: None, @@ -762,6 +776,7 @@ mod tests { storage: HashMap::new(), block_hashes: HashMap::new(), storage_cleared: std::collections::HashSet::new(), + accounts_not_existing: std::collections::HashSet::new(), code_by_hash: HashMap::new(), block_number: None, basefee: None, @@ -795,6 +810,7 @@ mod tests { storage: HashMap::new(), block_hashes: HashMap::new(), storage_cleared: std::collections::HashSet::new(), + accounts_not_existing: std::collections::HashSet::new(), code_by_hash, block_number: None, basefee: None, @@ -821,6 +837,7 @@ mod tests { accounts: HashMap::new(), storage: HashMap::new(), storage_cleared: std::collections::HashSet::new(), + accounts_not_existing: std::collections::HashSet::new(), block_hashes, code_by_hash: HashMap::new(), block_number: None, diff --git a/src/cache/snapshot.rs b/src/cache/snapshot.rs index 3019ec6..c6e8011 100644 --- a/src/cache/snapshot.rs +++ b/src/cache/snapshot.rs @@ -50,6 +50,11 @@ pub struct EvmSnapshot { /// ZERO and must NOT fall through to an `ext_db`, mirroring the live EVM SLOAD /// and [`EvmCache::cached_storage_value`](super::EvmCache::cached_storage_value). pub(crate) storage_cleared: HashSet
, + /// Accounts that are absent to the EVM (revm `NotExisting`): `basic` returns + /// `None` for them and must NOT fall through to an `ext_db`, mirroring revm + /// `DbAccount::info()` and [`EvmCache`](super::EvmCache)'s live account read. + /// These addresses are excluded from `accounts` / `code_by_hash`. + pub(crate) accounts_not_existing: HashSet
, pub(crate) block_hashes: HashMap, /// Bytecode lookup by code_hash (derived from accounts at creation time). pub(crate) code_by_hash: HashMap, @@ -106,6 +111,7 @@ mod tests { accounts: HashMap::new(), storage: HashMap::new(), storage_cleared: HashSet::new(), + accounts_not_existing: HashSet::new(), block_hashes: HashMap::new(), code_by_hash: HashMap::new(), block_number: Some(100), diff --git a/src/freshness.rs b/src/freshness.rs index 0fce064..5d5a8a7 100644 --- a/src/freshness.rs +++ b/src/freshness.rs @@ -908,9 +908,42 @@ struct ValidatorInput { /// Maximum fixed-point iterations the background validator performs while a /// correction keeps expanding a sim's volatile read set. A backstop against /// pathological contracts that read an unbounded chain of new volatile slots; -/// reaching it yields a best-effort `Corrected` (logged via `tracing::warn!`). +/// reaching it yields [`Validation::Unverified`] (the results have not reached a +/// verified fixed point, so they must not be trusted), logged via `tracing::warn!`. const MAX_VALIDATION_ROUNDS: u32 = 8; +/// Collect batch-fetcher results into a lookup map, requiring **every** requested +/// `(address, slot)` to be present and `Ok`. +/// +/// The validator must never silently trust a gap: a fetch error *or* a slot the +/// fetcher omitted from its response yields `Err(reason)` (mapped to +/// [`Validation::Unverified`] by the caller) rather than defaulting the missing +/// value to zero — a custom fetcher that drops a slot would otherwise produce a +/// false confirmation or correction. +fn collect_fetch_results( + requested: &[(Address, U256)], + results: Vec<(Address, U256, anyhow::Result)>, +) -> Result, String> { + let mut map: HashMap<(Address, U256), U256> = HashMap::new(); + for (addr, slot, value) in results { + match value { + Ok(v) => { + map.insert((addr, slot), v); + } + Err(e) => return Err(format!("fetch failed for {addr}:{slot}: {e}")), + } + } + for &key in requested { + if !map.contains_key(&key) { + return Err(format!( + "fetcher omitted requested slot {}:{}", + key.0, key.1 + )); + } + } + Ok(map) +} + /// The background validation routine. Touches only `Send` data — never the cache. fn run_validator(input: ValidatorInput) -> Validation { let ValidatorInput { @@ -961,21 +994,13 @@ fn run_validator(input: ValidatorInput) -> Validation { return Validation::Confirmed; } - // Fetch fresh values. Any error → Unverified (never trust silently). + // Fetch fresh values. Any error OR any omitted slot → Unverified (never trust + // silently: a missing result must not default to zero). let results = (fetcher)(verify.clone(), validation_block); - let mut fresh: HashMap<(Address, U256), U256> = HashMap::new(); - for (addr, slot, value) in results { - match value { - Ok(v) => { - fresh.insert((addr, slot), v); - } - Err(e) => { - return Validation::Unverified { - reason: format!("fetch failed for {addr}:{slot}: {e}"), - }; - } - } - } + let fresh = match collect_fetch_results(&verify, results) { + Ok(map) => map, + Err(reason) => return Validation::Unverified { reason }, + }; // Checkpoint: cancelled after the fetch returned but before we record any // observations or queue a correction. A cancel seen here discards the @@ -991,7 +1016,8 @@ fn run_validator(input: ValidatorInput) -> Validation { { let mut tracker = tracker.lock().unwrap_or_else(|e| e.into_inner()); for &(addr, slot) in &verify { - let new = fresh.get(&(addr, slot)).copied().unwrap_or(U256::ZERO); + // `collect_fetch_results` guarantees every requested slot is present. + let new = fresh[&(addr, slot)]; let old = snapshot.storage_value(addr, slot).unwrap_or(U256::ZERO); tracker.observe(addr, slot, new, now); if new != old { @@ -1047,26 +1073,35 @@ fn run_validator(input: ValidatorInput) -> Validation { for &(addr, slot, value) in &overrides { overlay.override_slot(addr, slot, value); } - if let Ok((result, access)) = overlay.call_raw_with_access_list_with( + // A host/transact error means the corrected re-run could not execute; + // we must not keep the stale optimistic result and call it "Corrected". + // (A revert/halt is `Ok(..)`, not an `Err`.) → Unverified. + let (result, access) = match overlay.call_raw_with_access_list_with( req.from, req.to, req.calldata.clone(), &req.tx, ) { - results[i] = result_to_sim(result, &access.to_eip2930()); - let new_volatile: Vec<(Address, U256)> = access - .slots - .iter() - .copied() - .filter(|(a, s)| registry.is_volatile(*a, *s, now)) - .collect(); - for &key in &new_volatile { - if !verified.contains(&key) { - new_candidates.insert(key); - } + Ok(v) => v, + Err(e) => { + return Validation::Unverified { + reason: format!("corrected re-run failed for request {i}: {e}"), + }; + } + }; + results[i] = result_to_sim(result, &access.to_eip2930()); + let new_volatile: Vec<(Address, U256)> = access + .slots + .iter() + .copied() + .filter(|(a, s)| registry.is_volatile(*a, *s, now)) + .collect(); + for &key in &new_volatile { + if !verified.contains(&key) { + new_candidates.insert(key); } - sim_reads[i] = new_volatile; } + sim_reads[i] = new_volatile; } // No sim read a changed slot (the change came from the predicted @@ -1075,14 +1110,21 @@ fn run_validator(input: ValidatorInput) -> Validation { if !any_rerun || new_candidates.is_empty() { break; } - // Results already reflect every override applied so far. Stop here rather - // than expanding the verified set further when the cap is reached. + // The fixed point was not reached within the cap: corrections kept opening + // new volatile slots. The results still rest on un-verified state, so we + // must NOT return a trusted `Corrected`. Return `Unverified` without + // queuing any pending corrections (matching the fetch-error paths); the + // still-volatile slots are re-discovered and re-fetched on the next run. if round >= MAX_VALIDATION_ROUNDS { tracing::warn!( rounds = round, - "freshness validator hit fixed-point iteration cap; returning best-effort Corrected" + "freshness validator exceeded fixed-point round cap; returning Unverified" ); - break; + return Validation::Unverified { + reason: format!( + "freshness validation exceeded fixed-point round cap ({MAX_VALIDATION_ROUNDS})" + ), + }; } // Checkpoint: cancelled mid-loop. Results so far reflect the applied @@ -1091,22 +1133,14 @@ fn run_validator(input: ValidatorInput) -> Validation { return Validation::Confirmed; } - // Fetch the newly-discovered candidates; any error → Unverified. + // Fetch the newly-discovered candidates; any error OR omitted slot → + // Unverified (a missing result must not default to zero). let new_vec: Vec<(Address, U256)> = new_candidates.into_iter().collect(); let fetched = (fetcher)(new_vec.clone(), validation_block); - let mut new_fresh: HashMap<(Address, U256), U256> = HashMap::new(); - for (addr, slot, value) in fetched { - match value { - Ok(v) => { - new_fresh.insert((addr, slot), v); - } - Err(e) => { - return Validation::Unverified { - reason: format!("fetch failed for {addr}:{slot}: {e}"), - }; - } - } - } + let new_fresh = match collect_fetch_results(&new_vec, fetched) { + Ok(map) => map, + Err(reason) => return Validation::Unverified { reason }, + }; // Diff + observe the newly fetched slots, growing the changed set. let mut grew = false; @@ -1114,7 +1148,8 @@ fn run_validator(input: ValidatorInput) -> Validation { let mut tracker = tracker.lock().unwrap_or_else(|e| e.into_inner()); for &(addr, slot) in &new_vec { verified.insert((addr, slot)); - let new = new_fresh.get(&(addr, slot)).copied().unwrap_or(U256::ZERO); + // `collect_fetch_results` guarantees every requested slot is present. + let new = new_fresh[&(addr, slot)]; let old = snapshot.storage_value(addr, slot).unwrap_or(U256::ZERO); tracker.observe(addr, slot, new, now); if new != old { diff --git a/src/state_update.rs b/src/state_update.rs index 8fc18b7..b448107 100644 --- a/src/state_update.rs +++ b/src/state_update.rs @@ -364,6 +364,17 @@ impl AccountPatch { /// /// The enum is `#[non_exhaustive]`: new scopes may be added pre-1.0 without a /// breaking change. +/// +/// # `StorageCleared`/`NotExisting` accounts +/// Purging *storage* ([`AllStorage`](Self::AllStorage) / [`Slots`](Self::Slots)) +/// removes the slot from the backend so a normal forked account re-fetches it on +/// the next read. For an account revm marks `StorageCleared`/`NotExisting` (a +/// locally-created/cleared account, e.g. after a `CREATE`/selfdestruct), the EVM +/// reads a missing slot as **zero without re-fetching** — its storage is locally +/// complete — so a purged slot reads `0`, not a fresh RPC value. This is correct +/// (such an account has no on-chain storage to refetch), but it means +/// [`Slots`](Self::Slots) does not force a refetch for those accounts. Use +/// [`Account`](Self::Account) (or `purge_account`) to fully drop the account. #[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[non_exhaustive] pub enum PurgeScope { diff --git a/tests/freshness.rs b/tests/freshness.rs index 9cc8c21..353f827 100644 --- a/tests/freshness.rs +++ b/tests/freshness.rs @@ -1716,3 +1716,122 @@ async fn on_new_block_ages_valid_through() -> Result<()> { ); Ok(()) } + +// =========================================================================== +// Phase 2 review (trust-contract hardening): the validator must NEVER return a +// trusted verdict on incomplete/ambiguous verification. +// =========================================================================== + +/// P2: a custom fetcher that OMITS a requested slot must yield `Unverified`, not +/// a false `Confirmed`/`Corrected` (missing results must not default to zero). +#[tokio::test(flavor = "multi_thread")] +async fn run_unverified_when_fetcher_omits_requested_slot() -> Result<()> { + let token = Address::repeat_byte(0x11); + let owner = Address::repeat_byte(0x22); + let recipient = Address::repeat_byte(0x33); + + let mut cache = cache_with_balance(token, owner, U256::from(1000)).await?; + // A fetcher that returns NOTHING — it omits every requested slot. + cache.set_storage_batch_fetcher(Arc::new( + |_req: Vec<(Address, U256)>, _block: Option| { + Vec::<(Address, U256, Result)>::new() + }, + )); + + let mut controller = FreshnessController::new(FreshnessRegistry::new(), AlwaysVerify); + let req = SimRequest::new(owner, token, transfer_calldata(recipient, U256::from(100))); + let sim = controller.run(&mut cache, vec![req])?; + + let validation = sim.validate().await; + assert!( + matches!(validation, Validation::Unverified { .. }), + "a fetcher that omits a requested slot must yield Unverified, not a false \ + confirmation/correction: {validation:?}" + ); + assert_eq!( + controller.pending_len(), + 0, + "Unverified must not queue any correction" + ); + Ok(()) +} + +/// Build runtime bytecode that reads slots `0..n` in order, returning the first +/// nonzero one (else zero). Reading slot `i+1` is gated on slot `i` being zero, so +/// each correction (slot → 0) opens exactly one new volatile slot — driving the +/// validator's fixed-point loop one round deeper per correction. +fn chained_sload_bytecode(n: u8) -> Bytes { + let ret_dest = 8u16 * (n as u16) + 2; // JUMPDEST offset (after the chain + PUSH1 0) + assert!(ret_dest <= 255, "return dest must fit in PUSH1"); + let ret = ret_dest as u8; + let mut code = Vec::new(); + for i in 0..n { + code.extend_from_slice(&[0x60, i, 0x54, 0x80, 0x60, ret, 0x57, 0x50]); + // PUSH1 i; SLOAD; DUP1; PUSH1 ret; JUMPI (if nonzero -> return it); POP + } + code.extend_from_slice(&[0x60, 0x00]); // all zero: PUSH1 0 + code.extend_from_slice(&[0x5b, 0x60, 0x00, 0x52, 0x60, 0x20, 0x60, 0x00, 0xf3]); + // JUMPDEST; PUSH1 0; MSTORE; PUSH1 0x20; PUSH1 0; RETURN (store TOS, return 32 bytes) + Bytes::from(code) +} + +/// P1: when corrections keep opening new volatile slots past +/// `MAX_VALIDATION_ROUNDS`, the validator must return `Unverified` — NOT a +/// best-effort (trusted) `Corrected` resting on un-verified state — and must +/// queue no corrections. +#[tokio::test(flavor = "multi_thread")] +async fn run_unverified_when_fixed_point_round_cap_exceeded() -> Result<()> { + use revm::state::{AccountInfo, Bytecode}; + + let mut cache = setup_cache().await?; + install_default_account(&mut cache, Address::ZERO); + let caller = Address::repeat_byte(0x66); + install_default_account(&mut cache, caller); + + // A 12-deep chain: each corrected slot opens the next, so the loop needs one + // round per slot — exceeding the 8-round cap well before the chain runs out. + let contract = Address::repeat_byte(0x55); + let code = Bytecode::new_raw(chained_sload_bytecode(12)); + let code_hash = code.hash_slow(); + cache.db_mut().insert_account_info( + contract, + AccountInfo { + balance: U256::ZERO, + nonce: 0, + code: Some(code), + code_hash, + account_id: None, + }, + ); + cache + .db_mut() + .replace_account_storage(contract, Default::default()) + .unwrap(); + // Snapshot: slots 0..12 all nonzero (EVM-visible overlay seed). The optimistic + // run reads only slot 0 (nonzero → returns). + for i in 0..12u64 { + cache + .db_mut() + .insert_account_storage(contract, U256::from(i), U256::from(1))?; + } + // Fresh chain: every slot dropped to 0 (stub returns 0 for all), so each + // correction flips the next branch and the loop never reaches a fixed point. + cache.set_storage_batch_fetcher(stub_fetcher(HashMap::new())); + + let mut controller = FreshnessController::new(FreshnessRegistry::new(), AlwaysVerify); + let req = SimRequest::new(caller, contract, Bytes::new()); + let sim = controller.run(&mut cache, vec![req])?; + + let validation = sim.validate().await; + assert!( + matches!(validation, Validation::Unverified { .. }), + "exceeding the fixed-point round cap must yield Unverified, not a trusted \ + Corrected: {validation:?}" + ); + assert_eq!( + controller.pending_len(), + 0, + "an Unverified (cap-exceeded) validation must queue no corrections" + ); + Ok(()) +} diff --git a/tests/snapshot_overlay.rs b/tests/snapshot_overlay.rs index cc3a510..10ae6d4 100644 --- a/tests/snapshot_overlay.rs +++ b/tests/snapshot_overlay.rs @@ -214,3 +214,47 @@ async fn snapshot_mirrors_live_read_for_cleared_account() -> Result<()> { ); Ok(()) } + +/// Regression (round-2 HIGH, account axis): `create_snapshot` / `EvmOverlay::basic` +/// must mirror the live account read for a `NotExisting` account. revm treats such +/// an account as absent (`DbAccount::info()` → None), and `loaded_account_info` +/// already does; the snapshot/parallel path must agree — not surface a phantom +/// existing account with stale info. Pre-fix `EvmOverlay::basic` returned +/// `Some(info)`. +#[tokio::test] +async fn snapshot_basic_returns_none_for_notexisting_account() -> Result<()> { + use revm::database::AccountState; + use revm::database_interface::Database; + use revm::state::AccountInfo; + + let acct = Address::repeat_byte(0x6e); + let mut cache = setup_cache().await?; + // An overlay account revm marks NotExisting (e.g. after a selfdestruct) carries + // (default) info but is absent to the EVM. + cache.db_mut().insert_account_info( + acct, + AccountInfo { + balance: U256::from(1000), + ..Default::default() + }, + ); + cache + .db_mut() + .cache + .accounts + .get_mut(&acct) + .expect("overlay account present") + .account_state = AccountState::NotExisting; + + let snapshot: Arc = cache.create_snapshot(); + let mut overlay = EvmOverlay::new(Arc::clone(&snapshot), None); + let basic = overlay + .basic(acct) + .map_err(|e| anyhow!("overlay basic read failed: {e:?}"))?; + assert!( + basic.is_none(), + "snapshot-backed overlay must read a NotExisting account as absent (None), \ + not a phantom Some(info); got {basic:?}" + ); + Ok(()) +} From 286d5e4926795891e84a63ce4e7e1744586c627e Mon Sep 17 00:00:00 2001 From: Kai Aldag Date: Tue, 16 Jun 2026 12:48:29 +0100 Subject: [PATCH 14/26] Phase 4: add event-pipeline spec (Pillar B.2) Build contract for the reader half of Pillar B: an EventDecoder trait + StateView, a DecoderRegistry, an ERC-20 Transfer decoder, a UniswapV3 Swap/Mint/Burn adapter, and the EventPipeline (ingest_logs / reorg_to / reconcile) that drives reactive cache updates. Adds the cold-aware StateUpdate::SlotMasked vocabulary variant so a pure decoder can express a partial update to a packed storage word (V3 slot0) without clobbering bits it does not own. Decisions locked with the user 2026-06-16 (SlotMasked; full Swap+Mint/Burn V3 coverage; purge-and-resync reorgs; sampled correct+alarm reconciliation). Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/phase-4-spec.md | 728 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 728 insertions(+) create mode 100644 docs/phase-4-spec.md diff --git a/docs/phase-4-spec.md b/docs/phase-4-spec.md new file mode 100644 index 0000000..78d2d6d --- /dev/null +++ b/docs/phase-4-spec.md @@ -0,0 +1,728 @@ +# Phase 4 implementation spec — event pipeline + adapters (Pillar B.2) + +Implementation contract for the **reader half** of Pillar B: turn an on-chain +`Log` into the Phase 3 [`StateUpdate`] vocabulary, apply it through +`apply_updates`, and keep the cache **reactively fresh** from the event stream — +with reconciliation, reorg handling, and freshness wiring. Read this **with** +[`ROADMAP.md`](ROADMAP.md) (the "Phase 4" row, the "Pillar B — event → state +pipeline" section, and the "Hard problems to resolve" list) and +[`phase-3-spec.md`](phase-3-spec.md) (the writer half this builds on). This +document is the precise build contract; where they overlap, prefer this. + +Phase 3 built the writer half (`StateUpdate` + `apply_updates` with cold-aware +`SlotDelta`/`BalanceDelta` RMW and `account_state`-correct reads). Phase 4 builds +the decoder, the protocol adapters, and the orchestration that drives them. + +## 0. Ground rules (non-negotiable) + +- **Branch:** create `phase-4-event-pipeline` off the current + `phase-3-state-updates` HEAD. Commit there in logical steps. Do **not** push, + do **not** tag, do **not** open a PR (the overseer does that). Commits must be + **unsigned**: `git -c commit.gpgsign=false commit …` (the 1Password signing + agent is unavailable here). End every commit message with exactly: + `Co-Authored-By: Claude Opus 4.8 (1M context) ` +- **Generic core vs `protocols`.** The pipeline, the `EventDecoder`/`StateView` + traits, the `DecoderRegistry`, the ERC-20 decoder, and the `SlotMasked` + vocabulary addition are **generic core** — they must compile and lint with + `--no-default-features`. Only the UniswapV3 adapter (`uniswap_v3`) is gated + behind the `protocols` feature. +- **Green bar at every commit, both feature configs:** + - `cargo fmt --all --check` + - `cargo clippy --all-targets --no-deps -- -D warnings` + - `cargo clippy --lib --no-default-features --no-deps -- -D warnings` + - `cargo test` + - `RUSTDOCFLAGS="-D warnings" cargo doc --no-deps` + - `cargo bench --no-run` (all benches build offline) +- MSRV is 1.88 — no newer-than-1.88 std APIs. Edition 2024. +- **Do not break existing behavior or any existing test.** The Phase 3 surface + (`StateUpdate`, `apply_updates`, `modify_slot`, `StateDiff`) keeps its shape; + Phase 4 *adds* the `SlotMasked` variant and the `skipped_masks` diff field + under the pre-1.0 break policy (both already-`#[non_exhaustive]` types). +- **No new dependencies.** `alloy-primitives` (with `Log`), `alloy-sol-types` + (`sol!` / `SolEvent`), `alloy-provider`, `futures`, and `tokio` are already + present. Decode logs with `sol!`-generated event types + `SolEvent`, not + hand-rolled byte slicing. + +## 1. Objective & scope + +Today the crate can *write* targeted state but has no way to *derive* those +writes from chain activity: a caller must hand it concrete `StateUpdate`s. Phase +4 closes the loop — decode a `Log` into `StateUpdate`s, apply them, and run the +reactive maintenance (reconcile, reorg) that keeps event-derived state honest. + +**In scope:** + +1. **`StateUpdate::SlotMasked`** — a cold-aware read-modify-write *masked* slot + write (`(old & !mask) | (value & mask)`), so a pure decoder can express a + partial update to a **packed** storage word (e.g. V3 `slot0`) without knowing + or clobbering the bits it does not own. Generic core (§4.1). +2. **`EventDecoder` + `StateView`** — the decoder trait (`Log` + read-only + pre-state view → `Vec`) and the narrow read-only cache view it is + handed. `EvmCache` implements `StateView`. Generic core (§4.2). +3. **`DecoderRegistry`** — dispatches a log to the decoder(s) registered for its + emitting address (and/or topic0) and concatenates their output. Generic core + (§4.3). +4. **`Erc20TransferDecoder`** — generic ERC-20 `Transfer` → relative balance + `SlotDelta`s (the §15 reactive-balance case, now log-driven). Generic core + (§5). +5. **`UniswapV3Decoder`** — `protocols`-gated adapter: `Swap` → `slot0` + (masked sqrtPriceX96 + tick) + `liquidity`; `Mint`/`Burn` → per-tick + `liquidityGross`/`liquidityNet`, the `initialized` flag, `tickBitmap` word + flips, and the global `liquidity` (conditional on the current tick). Computed + against the `StateView` (tick maintenance is inherently RMW). (§6). +6. **`EventPipeline`** — the orchestration: `ingest_logs` (decode+apply + **log-by-log**, in order, recording touched state for reorg tracking), + `reorg_to` (purge-and-resync addresses touched after the new head), and + `reconcile` (sampled RPC re-read via `verify_slots`: correct **and** alarm). + Generic core (§7). +7. **Freshness wiring** — `BlockDigest` surfaces the touched `(address, slot)` + set so a caller can classify event-derived slots (`valid_through` / `pin`) and + call `FreshnessController::on_new_block`. No controller internals change (§8). +8. Offline example, benchmark, docs, CHANGELOG, ROADMAP → Done (§11). + +**Out of scope (document as follow-ups; do not build):** +- **A concrete WS transport / live subscription loop.** The async `drive` + convenience (§7.5) is generic over a log source and is exercised only by the + offline example feeding a vec-backed source; a production WS/`subscribe_logs` + adapter is a follow-up. The *tested* surface is the synchronous core. +- **V3 fee-growth / oracle observation maintenance.** Event-derived tick init + does **not** reconstruct `feeGrowthOutside0/1X128` (slots +1/+2) or oracle + observations — those are not derivable from `Mint`/`Burn`/`Swap`. Swap + *price/liquidity quoting* is unaffected; fee-accounting reads are not + maintained. Document as a KNOWN_ISSUE with reconcile/purge as the backstop + (§6.4). +- **Non-Uniswap-layout V3 (Slipstream slot0).** The adapter assumes the + Uniswap/Pancake `slot0` bit layout (only base slots differ). Slipstream's + different `slot0` packing is a follow-up. +- **COW snapshots** (Phase 5). The pipeline mutates the existing `EvmCache` + layers via `apply_updates`. + +## 2. Reuse these existing pieces (do not reinvent) + +- **`StateUpdate` / `apply_update` / `apply_updates` / `modify_slot`** + (`state_update.rs`, `cache/mod.rs`) — the write half. The pipeline applies + decoded updates through `apply_updates`; the `SlotMasked` handler reuses the + private `write_slot_through` and the `account_state`-aware + `cached_storage_value` (§16.0). +- **`EvmCache::cached_storage_value`** — the `StateView::storage` + implementation (overlay ▸ backend ▸ `None`, `account_state`-correct). +- **`EvmCache::verify_slots`** (`cache/mod.rs`) — the synchronous + fetch-compare-inject reconciliation primitive. `reconcile` is a thin wrapper: + it samples event-derived slots and calls `verify_slots`; the returned + `Vec` is the drift report (verify_slots already injected the fresh + chain values — correct + alarm). +- **`EvmCache::purge_account` / `apply_update(Purge { … })`** — the reorg + purge mechanism. `reorg_to` purges touched addresses through `apply_updates` + of `Purge` updates so the next read re-fetches. +- **`inspector::TransferInspector::parse_transfer`** + the + `TRANSFER_EVENT_SIGNATURE` constant (`src/inspector.rs`) — reuse for the + ERC-20 decoder's signature match and topic decoding (or reuse the same + `sol!` event). Do not redefine the signature constant. +- **`cache::storage_keys`** (`protocols`) — `V3_*`/`PANCAKE_V3_*` slot + constants, `v3_tick_info_storage_keys_with_base`, + `v3_tick_bitmap_storage_key_with_base`, `i256_from_i24`, `i128_to_u256`. The + V3 adapter reuses these for slot derivation and packing. +- **`freshness::{FreshnessController, FreshnessRegistry, Validity, SlotChange}`** + — the freshness wiring target. `SlotChange` is the reconcile-report element. +- **`alloy_sol_types::{sol, SolEvent}`** — generate `Swap`/`Mint`/`Burn` and + ERC-20 `Transfer` event types and decode with `SolEvent::decode_log_data`. +- **The offline harness** — `examples/support/mock.rs` (`offline_cache`, + `install_mock_erc20`, `MockERC20`, `MOCK_ERC20_BALANCE_SLOT`), + `tests/common`. The new tests/example build the cache over the mocked provider + and never touch the network. + +## 3. Module layout + +A new `src/events/` directory module (the crate's only other dir module is +`cache/`): + +- **`src/events/mod.rs`** (generic core): `EventDecoder`, `StateView`, + `DecoderRegistry`, `EventPipeline`, `BlockDigest`, `ReconcileReport`, + `ReorgConfig`, and the async `drive` convenience + its `LogSource` trait. The + module `//!` doc frames Pillar B.2 and the `!Send`-cache discipline. +- **`src/events/erc20.rs`** (generic core): `Erc20TransferDecoder` + config. +- **`src/events/uniswap_v3.rs`** (`#[cfg(feature = "protocols")]`): + `UniswapV3Decoder` + `UniswapV3Layout` config (base slots + tick spacing). +- **`src/state_update.rs`**: add the `SlotMasked` variant, the `slot_masked` + constructor, `SkippedMask`, and the `StateDiff.skipped_masks` field + + `merge`/`has_skipped`/`skipped_len` updates. +- **`src/cache/mod.rs`**: the `SlotMasked` apply arm (reusing `write_slot_through` + + the cold-aware read); `impl events::StateView for EvmCache`. +- **`src/lib.rs`**: `pub mod events;` + re-exports (§9). + +## 4. Core types & behavior + +### 4.1 `StateUpdate::SlotMasked` — cold-aware masked write (generic core) + +```rust +pub enum StateUpdate { + Slot { address, slot, value }, + SlotDelta { address, slot, delta }, + /// Set only the `mask` bits of a storage slot to the corresponding bits of + /// `value`, preserving the rest: `new = (old & !mask) | (value & mask)`. + /// Read-modify-write, **cold-aware** — a masked write to a slot absent from + /// both layers is not applied (the un-masked bits are unknown); it is + /// surfaced in [`StateDiff::skipped_masks`]. + SlotMasked { address: Address, slot: U256, mask: U256, value: U256 }, // NEW + BalanceDelta { address, delta }, + Account { address, patch }, + Purge { address, scope }, +} +impl StateUpdate { + pub fn slot_masked(address: Address, slot: U256, mask: U256, value: U256) -> Self; +} + +/// A masked write ([`StateUpdate::SlotMasked`]) skipped because the target slot +/// was cold (un-masked bits unknown). Fetch+seed the slot, then retry. +#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct SkippedMask { pub address: Address, pub slot: U256, pub mask: U256, pub value: U256 } + +pub struct StateDiff { + pub slots: Vec, + pub accounts: Vec, + pub purged: Vec, + pub skipped: Vec, + pub skipped_balances: Vec, + pub skipped_masks: Vec, // NEW +} +``` + +`SkippedMask` is a leaf record constructed as a struct literal in equality +assertions, so — like `SkippedDelta`/`SkippedBalanceDelta` (§16.4) — it is +**not** `#[non_exhaustive]`. `StateUpdate` and `StateDiff` already are. + +Apply behavior (`cache/mod.rs`, mirrors the `SlotDelta` arm): +- Read `old = cached_storage_value(address, slot)` (cold-aware, §16.0). +- If `Some(old)`: `new = (old & !mask) | (value & mask)`; write through both + layers via `write_slot_through`; push a `SlotChange { old, new }` iff + `old != new`. +- If `None` (cold): push `SkippedMask` to `diff.skipped_masks`, write nothing. + +`StateDiff::merge` extends `skipped_masks`. `has_skipped` /`skipped_len` +include `skipped_masks`. `is_empty`/`len` stay **changes-only** (a skip is not a +change). serde derives on `SkippedMask`. Module `//!` doc gains a `SlotMasked` +paragraph (packed-word updates, cold-aware). + +> A masked write with `mask == U256::MAX` equals an absolute `Slot` write but +> **stays cold-skip** (a cold full-mask write is still skipped, unlike `Slot` +> which writes unconditionally). Decoders that want an unconditional absolute +> write use `Slot`; those that must preserve neighbouring bits use `SlotMasked`. + +### 4.2 `EventDecoder` + `StateView` + +```rust +/// Read-only view of current cached state handed to a decoder. +/// +/// Decoders that compute post-state from pre-state (e.g. V3 tick maintenance) +/// read through this; stateless decoders (ERC-20 `Transfer`, V3 `Swap`) ignore +/// it. The view never touches RPC — a slot absent from the cache reads `None`. +pub trait StateView { + /// Current cached value of `(address, slot)` (overlay ▸ backend ▸ `None`), + /// matching what the EVM would `SLOAD` (`account_state`-aware). + fn storage(&self, address: Address, slot: U256) -> Option; +} + +/// Decode one log into zero or more targeted [`StateUpdate`]s. +/// +/// `decode` is a pure function of `(log, pre-state)`: it performs no I/O and +/// emits data (the updates are serializable and replayable against matching +/// pre-state). The pipeline applies the result through `apply_updates`. +pub trait EventDecoder: Send + Sync { + fn decode(&self, log: &Log, view: &dyn StateView) -> Vec; +} +``` + +`EvmCache` implements `StateView` via `cached_storage_value`. Rationale for the +`StateView` parameter (a refinement of the ROADMAP's `fn decode(&self, log) -> +Vec` sketch): event-driven sync is fundamentally *events + +pre-state → post-state*. Most updates are expressible without pre-state +(`SlotDelta` and `SlotMasked` are RMW **at apply time**), but V3 tick +maintenance must read current `liquidityGross`/`liquidityNet`/`tick`/bitmap to +compute the next packed value, and a pure decoder cannot. Handing decoders a +narrow read-only view keeps the output as serializable `StateUpdate` data while +making the stateful adapters expressible and offline-testable (feed a stub +view). + +### 4.3 `DecoderRegistry` + +```rust +#[derive(Default)] +pub struct DecoderRegistry { /* global decoders + per-address decoders */ } + +impl DecoderRegistry { + pub fn new() -> Self; + /// Register a decoder consulted for **every** log. + pub fn register(&mut self, decoder: Arc) -> &mut Self; + /// Register a decoder consulted only for logs emitted by `address`. + pub fn register_for_address(&mut self, address: Address, decoder: Arc) -> &mut Self; + /// Decode `log` through every applicable decoder, concatenating the results + /// (address-scoped decoders first, then global), preserving order. + pub fn decode(&self, log: &Log, view: &dyn StateView) -> Vec; +} +``` + +Dispatch is by emitting address (`log.address`); topic0 filtering is the +decoder's own concern (each decoder returns `vec![]` for a log it does not +recognise). Keep it simple: address-scoped entries + a global list, both +consulted, output concatenated. + +## 5. `Erc20TransferDecoder` (generic core, `events/erc20.rs`) + +```rust +pub struct Erc20TransferDecoder { + /// Balance mapping slot per token (the `balanceOf` mapping's base slot). + balance_slots: HashMap, + /// Fallback balance slot for tokens not in the map. + default_balance_slot: U256, +} +impl Erc20TransferDecoder { + pub fn new(default_balance_slot: U256) -> Self; + pub fn with_token(mut self, token: Address, balance_slot: U256) -> Self; +} +impl EventDecoder for Erc20TransferDecoder { /* … */ } +``` + +Decode rule for a `Transfer(from, to, value)` log (signature match via +`TRANSFER_EVENT_SIGNATURE`; topics/data decoded like `parse_transfer`): +- `slot = balance_slots.get(token).copied().unwrap_or(default_balance_slot)`. +- `balance_key(owner) = U256::from(keccak256(abi_encode((owner, slot))))`. +- Emit, **skipping the zero-address leg** (mint = `from == 0`, burn = `to == 0`): + - if `from != Address::ZERO`: `SlotDelta::Sub(value)` on `balance_key(from)`. + - if `to != Address::ZERO`: `SlotDelta::Add(value)` on `balance_key(to)`. +- A non-`Transfer` log (wrong topic0, < 3 topics, < 32 data bytes) → `vec![]`. + +Cold balances follow the Phase 3 contract: the `SlotDelta` is skipped and +surfaced in `StateDiff.skipped` (the caller seeds the balance, or the next read +lazily fetches it). The decoder ignores the `StateView`. `value == 0` transfers +emit deltas of zero (a no-op at apply — empty diff); that is acceptable. + +## 6. `UniswapV3Decoder` (`protocols`, `events/uniswap_v3.rs`) + +```rust +#[derive(Clone, Debug)] +pub struct UniswapV3Layout { + pub slot0_slot: U256, // V3_SLOT0_SLOT (0) — Uniswap/Pancake + pub liquidity_slot: U256, // V3_LIQUIDITY_SLOT (4) / PANCAKE (5) + pub ticks_base_slot: U256, // V3_TICKS_BASE_SLOT (5) / PANCAKE (6) + pub tick_bitmap_base_slot: U256, // V3_TICK_BITMAP_BASE_SLOT (6) / PANCAKE (7) + pub tick_spacing: i32, // pool tickSpacing (for bitmap word/bit) +} +impl UniswapV3Layout { + pub fn uniswap(tick_spacing: i32) -> Self; // canonical Uniswap V3 slots + pub fn pancake(tick_spacing: i32) -> Self; // PancakeSwap V3 slots +} + +pub struct UniswapV3Decoder { + /// Per-pool layout (slot bases + tick spacing). A log from an unregistered + /// pool decodes to nothing. + pools: HashMap, +} +impl UniswapV3Decoder { + pub fn new() -> Self; + pub fn with_pool(mut self, pool: Address, layout: UniswapV3Layout) -> Self; +} +impl EventDecoder for UniswapV3Decoder { /* … */ } +``` + +`tick_spacing` is required for `Mint`/`Burn` bitmap maintenance: the tickBitmap +is keyed by the **compressed** tick `tick / tick_spacing`. A log from a pool not +in `pools` → `vec![]`. Match events by topic0 (`Swap`/`Mint`/`Burn` signature +hashes from `sol!`); decode with `SolEvent`. + +### 6.1 `Swap` → price + liquidity (stateless) + +`Swap(sender, recipient, amount0, amount1, sqrtPriceX96, liquidity, tick)`: +- **slot0** (`SlotMasked`, preserves observation/feeProtocol/`unlocked` bits): + - `mask = (U256::from(1) << 184) - 1` (low 184 bits = sqrtPriceX96 [0,160) + + tick [160,184)). + - `value = U256::from(sqrtPriceX96) | (tick_24bit << 160)` where `tick_24bit` + is the int24 two's-complement low-24-bits of `tick` + (`U256::from(tick as i32 as u32 & 0x00FF_FFFF)`). + - Emit `StateUpdate::slot_masked(pool, slot0_slot, mask, value)`. +- **liquidity** (absolute — the event carries the post-swap pool liquidity): + - `StateUpdate::slot(pool, liquidity_slot, U256::from(liquidity))`. + +The `unlocked` bit (bit 240) and observation/fee bits are **preserved** by the +mask — clobbering `unlocked` to 0 would make a subsequent quote/swap revert +`LOK`. This is the headline correctness reason for `SlotMasked`. Stateless +(ignores the view); a cold slot0 → `skipped_masks` (the pool must be seeded +first). + +### 6.2 `Mint` → tick + liquidity maintenance (stateful, reads `StateView`) + +`Mint(sender, owner, tickLower, tickUpper, amount, amount0, amount1)` adds +`amount` (uint128 liquidity) over `[tickLower, tickUpper)`. For **each** of +`tickLower` and `tickUpper`, and for the global liquidity, compute the post-state +from the current cached value (read via `view.storage`); emit an **absolute** +`Slot` write of the recomputed word (a packed word recomputed from known +pre-state is an absolute write, not a delta). If a needed word is **cold** +(`view.storage` → `None`), **skip that update and surface it** as a +`SkippedMask`/`SkippedDelta` (choose `SkippedDelta` with a zero-amount marker is +wrong — use a dedicated skip; see §6.5) so the caller knows the pool tick state +is incomplete (re-seed via `inject_v3_ticks`). + +Per tick (`tick` ∈ {`tickLower`, `tickUpper`}): +- **Tick slot +0** (`liquidityGross` [0,128) ‖ `liquidityNet` [128,256) signed): + - base = `v3_tick_info_storage_keys_with_base(tick, ticks_base_slot)[0]`. + - read current word; `gross = low128`, `net = high128 as i128`. + - `gross' = gross + amount` (uint128). + - `net' = net + amount` for `tickLower`, `net' = net - amount` for `tickUpper` + (int128). + - repacked = `U256::from(gross') | (i128_to_u256(net') << 128)`; emit + `Slot(pool, base, repacked)`. +- **Tick slot +3** (`initialized` flag, byte 31 / bit 248 — matching the + existing `inject_v3_ticks` placement): if `gross == 0 && gross' > 0` + (tick newly initialized), set `initialized` by emitting + `SlotMasked(pool, base+3, mask = U256::from(1) << 248, value = U256::from(1) << 248)`. + (No change if it was already initialized.) +- **tickBitmap**: when a tick is newly initialized, flip its bit: + - `compressed = tick / tick_spacing` (floor toward negative infinity — match + Solidity: `tick / tickSpacing` truncates toward zero, and V3 requires + `tick % tickSpacing == 0`, so plain integer division is exact). + - `word_pos = (compressed >> 8) as i16`, `bit_pos = (compressed & 0xFF) as u8`. + - key = `v3_tick_bitmap_storage_key_with_base(word_pos, tick_bitmap_base_slot)`. + - emit `SlotMasked(pool, key, mask = U256::from(1) << bit_pos, value = U256::from(1) << bit_pos)` + (set the bit). On Burn that uninitialises the tick, clear it (value = 0). + +Global **liquidity** (slot `liquidity_slot`): the `Mint` event does **not** +carry the resulting pool liquidity, so it must be derived: read current `slot0` +→ extract `tick` (bits [160,184), sign-extended int24); if +`tickLower <= currentTick < tickUpper`, read current `liquidity` and emit +`Slot(pool, liquidity_slot, current + amount)`. If `slot0` or `liquidity` is +cold, skip+surface. (Safety net: the next `Swap` sets `liquidity` absolutely.) + +### 6.3 `Burn` → the inverse + +`Burn(owner, tickLower, tickUpper, amount, amount0, amount1)`: identical to +`Mint` with the signs inverted: +- `gross' = gross - amount` (uint128, saturating at 0 defensively). +- `net' = net - amount` for `tickLower`, `net' = net + amount` for `tickUpper`. +- If `gross > 0 && gross' == 0` (tick now uninitialised): clear the + `initialized` flag (`SlotMasked` slot+3 value 0) **and** clear the bitmap bit + (`SlotMasked` value 0). +- Global liquidity: `Slot(pool, liquidity_slot, current - amount)` if current + tick in `[tickLower, tickUpper)`. + +> A `Burn` removing all of a tick's liquidity but a same-block re-`Mint` is +> handled by the **log-by-log** apply order (§7.1): the second decode reads the +> first's applied effect through the view. + +### 6.4 Known limitation (document, do not fix) + +Event-derived tick maintenance does **not** set `feeGrowthOutside0/1X128` +(slots +1/+2), `secondsOutside`, or oracle observations — these are not +derivable from `Mint`/`Burn`/`Swap`. **Swap price/liquidity quoting is +unaffected** (the swap-amount math does not depend on `feeGrowthOutside`); fee +accounting and `collect` are not maintained. Record this as a `KNOWN_ISSUES.md` +entry with sampled `reconcile` + reorg `purge` as the backstop, in the project's +honest-freshness spirit. + +### 6.5 Cold-skip surfacing for stateful V3 updates + +When a V3 tick/liquidity update cannot be computed because a needed word is cold, +surface it so the gap is visible (never silently drop it). Reuse +`SkippedMask` for masked sub-word updates (bitmap/initialized) and, for the +absolute tick-word / liquidity writes that were skipped, push a `SkippedMask` +with `mask == U256::MAX` and `value == U256::ZERO` as the "could-not-compute" +marker, **or** (cleaner) add the skipped target to a dedicated field. **Locked +choice:** reuse `SkippedMask` with `mask == U256::MAX, value == 0` as the +cold-tick marker to avoid a fourth skip vector; document this convention on +`SkippedMask`. The pipeline's `BlockDigest.skipped` count (via +`StateDiff::skipped_len`) then includes them, and the caller re-seeds the pool. + +## 7. `EventPipeline` (generic core, `events/mod.rs`) + +```rust +pub struct EventPipeline { + registry: DecoderRegistry, + reorg: ReorgConfig, + touched: VecDeque<(u64, Vec
)>, // ring of per-block touched addrs + derived_slots: HashSet<(Address, U256)>, // event-derived slots (for reconcile sampling) +} + +#[derive(Clone, Debug)] +pub struct ReorgConfig { + /// How many recent blocks of touched-address history to retain for reorg + /// purge (the reorg horizon). Older entries are dropped. + pub depth: usize, + /// Purge scope used on reorg (default `AllStorage` — storage re-fetches but + /// the account header survives; `Account` for a full drop). + pub scope: PurgeScope, +} + +pub struct BlockDigest { + pub block: u64, + /// Merged diff of everything applied for the block (changes-only + skips). + pub applied: StateDiff, + /// Number of logs that decoded to at least one update. + pub decoded_logs: usize, + /// The (address, slot) set written this block (for freshness classification). + pub touched_slots: Vec<(Address, U256)>, +} + +pub struct ReconcileReport { + pub checked: usize, + /// Slots whose event-derived value disagreed with chain truth. Non-empty = + /// drift alarm. `verify_slots` has already injected the fresh values. + pub mismatched: Vec, +} + +impl EventPipeline { + pub fn new(registry: DecoderRegistry) -> Self; // default ReorgConfig + pub fn with_reorg_config(mut self, cfg: ReorgConfig) -> Self; + + /// Decode + apply a block's logs, **log-by-log in order**, recording touched + /// state for reorg tracking. Returns the per-block digest. + pub fn ingest_logs(&mut self, cache: &mut EvmCache, block: u64, logs: &[Log]) -> BlockDigest; + + /// Reorg to `new_head`: purge (per `ReorgConfig.scope`) every address + /// touched in a block **>** `new_head`, drop those ring entries, and return + /// the merged purge diff. The next read re-fetches from RPC. + pub fn reorg_to(&mut self, cache: &mut EvmCache, new_head: u64) -> StateDiff; + + /// Sampled reconciliation: re-read `slots` via `EvmCache::verify_slots` + /// (correct + alarm). Returns the mismatches; an empty `slots` or no fetcher + /// surfaces as appropriate (errors if no fetcher, mirroring `verify_slots`). + pub fn reconcile(&mut self, cache: &mut EvmCache, slots: &[(Address, U256)]) -> Result; + + /// All event-derived slots seen so far (sampling source for `reconcile`). + pub fn derived_slots(&self) -> impl Iterator + '_; +} +``` + +### 7.1 `ingest_logs` — decode + apply **log-by-log** + +For each log, in order: `let updates = registry.decode(log, &*cache); let diff = +cache.apply_updates(&updates); merge into the block diff`. Apply **immediately +per log** (not decode-all-then-apply-all) so a later log's decode sees the +effects of earlier logs in the same block through the `StateView` (e.g. two +overlapping `Mint`s, or a `Burn`+`Mint` pair). Record the touched addresses +(`diff.slots` + `diff.accounts` addresses + `diff.skipped*` targets' addresses) +into the ring under `block`, and the touched `(address, slot)` into +`derived_slots`. Trim the ring to `ReorgConfig.depth`. + +> `&*cache` is used as the `&dyn StateView` while `cache.apply_updates(&mut …)` +> needs `&mut` — sequence them (decode borrow ends before the apply borrow), do +> not hold both. Decode returns owned `Vec`, so there is no +> borrow overlap. + +### 7.2 `reorg_to` — purge-and-resync + +Collect every address in ring entries with `block > new_head`; dedupe; for each, +`apply_update(Purge { address, scope: cfg.scope })`; remove those ring entries +and their `derived_slots`. Merge the purge `StateDiff`s and return. (The caller +then re-ingests the canonical chain's logs for the reorged range, and/or the +next read lazily re-fetches.) + +### 7.3 `reconcile` — correct + alarm + +`let changed = cache.verify_slots(slots)?;` → `ReconcileReport { checked: +slots.len(), mismatched: changed }`. `verify_slots` already injected the fresh +chain values (correct); the returned set is the alarm. Document that a non-empty +`mismatched` means event-derived state had drifted and has now been corrected. + +### 7.4 `!Send` discipline + +All three methods take `&mut EvmCache` and are **synchronous** — they never +`.await`, so the `!Send` cache is never held across a yield. This is what makes +the core deterministically testable offline. + +### 7.5 `drive` — async convenience (thin, example-only) + +A generic `LogSource` (`async fn next_block(&mut self) -> Option<(u64, Vec, ReorgSignal)>`) +and an `async fn drive(pipeline, cache, source, hooks)` that loops: pull a block, +`reorg_to` if signalled, `ingest_logs`, invoke an optional per-block hook (where +the caller wires `FreshnessController::on_new_block` + classification). Runs on +the current task (holds the `!Send` cache across the *source* await only — the +source future is `Send`; the cache is untouched during the await). **Not** +unit-tested beyond a vec-backed `LogSource` smoke test in the example; the +synchronous core (§7.1–7.3) is the contract. + +## 8. Freshness wiring (behavior-preserving) + +No change to `FreshnessController` internals. The integration is demonstrated, +not hard-wired: `BlockDigest.touched_slots` lets a caller mark event-derived +slots `Pinned` or `ValidThrough(block + horizon)` in a `FreshnessRegistry` (so +the optimistic validator does not waste RPC re-verifying state the pipeline keeps +fresh), then call `controller.on_new_block(block)`. The example shows this +end-to-end. Document the recommended pattern (event-driven slots → `Pinned`, +reconciled periodically) on the `EventPipeline` type. + +## 9. Public re-exports (`src/lib.rs`) + +```rust +pub mod events; +pub use events::{ + BlockDigest, DecoderRegistry, EventDecoder, EventPipeline, ReconcileReport, + ReorgConfig, StateView, +}; +pub use events::erc20::Erc20TransferDecoder; +#[cfg(feature = "protocols")] +pub use events::uniswap_v3::{UniswapV3Decoder, UniswapV3Layout}; +// state_update additions: +pub use state_update::{SkippedMask /* + existing */}; +``` + +## 10. Tests (offline, no network) — the acceptance contract + +Authored **before** implementation. In-module unit tests where pure; integration +tests in new `tests/event_pipeline.rs` (reuse `tests/common` + the `mock` +harness pattern). All offline. + +**`state_update.rs` unit (pure):** +- `slot_masked_constructor_produces_variant`. +- `state_diff_merge_extends_skipped_masks_without_counting_it`. +- `slot_masked` serde JSON round-trip; `SkippedMask` round-trip. +- `has_skipped`/`skipped_len`/`is_fully_applied` include `skipped_masks`. + +**`tests/state_update.rs` (masked apply, mocked cache):** +- `slot_masked_sets_only_masked_bits` — seed slot = `0xFFFF…FF00` (overlay), + `SlotMasked{ mask: 0xFF, value: 0x42 }` → `0xFFFF…FF42`; other bits preserved; + `SlotChange{old,new}` recorded. +- `slot_masked_noop_when_masked_bits_already_equal` → empty diff. +- `slot_masked_cold_slot_is_skipped_and_surfaced` → `diff.skipped_masks == + [SkippedMask{..}]`, slot still cold, `has_skipped()`. +- `slot_masked_writes_through_both_layers` — overlay-resident slot → both layers. +- `slot_masked_full_mask_equals_absolute_on_hot_but_skips_cold`. + +**`events` unit / `tests/event_pipeline.rs` (decoders + pipeline):** + +*Decoder purity & registry:* +- `decoder_registry_dispatches_by_address` — a decoder registered for token A + fires only for A's logs; a global decoder fires for all; output concatenated + in order. +- `unknown_log_decodes_to_empty` — non-matching topic0 → `vec![]`. + +*ERC-20 (`Erc20TransferDecoder`):* +- `erc20_transfer_decodes_to_sub_and_add_deltas` — `Transfer(A,B,100)` → + `[SlotDelta::Sub(100) @ balanceSlot(A), SlotDelta::Add(100) @ balanceSlot(B)]` + at the configured mapping slot. +- `erc20_mint_skips_zero_from` / `erc20_burn_skips_zero_to` — only the non-zero + leg emitted. +- `erc20_uses_per_token_slot_override_else_default`. +- `erc20_ingest_updates_balance_and_conserves` — **end-to-end**: build the mock + cache, seed two holders' balance slots (overlay-resident, EVM-visible), + `ingest_logs` a `Transfer` log, assert both balances via `balance_of` + (real `SLOAD`) and that `from + to` is conserved; `digest.applied.slots` has 2 + entries. +- `erc20_cold_balance_transfer_is_skipped_and_surfaced` — unseeded `to` → + `digest.applied.skipped` non-empty; `has_skipped()`. + +*UniswapV3 (`protocols`, gated tests):* +- `v3_swap_sets_price_and_tick_preserving_unlocked` — seed slot0 with a known + packed word incl. `unlocked=1` (bit 240) and a nonzero observation index; + ingest a `Swap` with new sqrtPriceX96/tick; assert slot0's low-184 bits are the + new price/tick **and** bits 184+ (incl. `unlocked`) are unchanged. +- `v3_swap_sets_liquidity_absolute` — `liquidity` slot == event liquidity. +- `v3_swap_cold_slot0_is_skipped` — unseeded slot0 → `skipped_masks`. +- `v3_mint_increments_gross_and_net_signs` — seed tick slot+0 = 0; `Mint(amount)` + at `[lo,hi]` → lo word `gross=amount, net=+amount`; hi word + `gross=amount, net=-amount` (decode the packed words). +- `v3_mint_initializes_tick_and_flips_bitmap` — newly-init tick sets slot+3 + initialized bit and flips the correct bitmap word/bit (using `tick_spacing`). +- `v3_burn_decrements_and_uninitializes` — `Burn` returning gross to 0 clears the + initialized bit and the bitmap bit. +- `v3_mint_updates_global_liquidity_when_in_range` — seed slot0 tick within + `[lo,hi)` and a known `liquidity`; `Mint` → liquidity += amount; out-of-range → + liquidity unchanged. +- `v3_mint_cold_tick_word_is_skipped` — unseeded tick word → surfaced skip, no + write. +- `v3_same_block_burn_then_mint_sees_prior_apply` — two logs in one + `ingest_logs`; the `Mint` decode reads the `Burn`'s applied gross/net. + +*Pipeline (reorg + reconcile):* +- `ingest_records_touched_and_trims_ring_to_depth`. +- `reorg_to_purges_addresses_touched_after_head` — ingest blocks N, N+1, N+2 + touching distinct pools; `reorg_to(N)` purges only N+1/N+2 pools (assert their + storage re-reads cold / re-fetches; N's survives). +- `reorg_to_returns_merged_purge_diff` — `PurgeRecord`s for the purged set. +- `reconcile_reports_mismatch_and_corrects` — stub the batch fetcher so an + event-derived slot disagrees; `reconcile` returns it in `mismatched` and the + cache now holds the fresh value (assert via `cached_storage_value`). +- `reconcile_empty_when_event_state_matches_chain`. +- `reconcile_errs_without_fetcher`. + +**Existing suites stay green** — `tests/state_update.rs`, `tests/freshness.rs`, +`tests/snapshot_overlay.rs`, the `protocols` cache tests. + +## 11. Docs, example & benchmark + +- **Example** `examples/reactive_cache.rs` (offline, `examples/support`): + build a `from_backend`/mock cache; register an `Erc20TransferDecoder` and a + `UniswapV3Decoder` in a `DecoderRegistry`; `ingest_logs` a small vec of logs + (an ERC-20 `Transfer` + a V3 `Swap`) for a block; print the `BlockDigest`; + then demonstrate (a) a `reorg_to` purge, and (b) a `reconcile` drift alarm + against a stub fetcher; wire `FreshnessController::on_new_block` + + `registry.valid_through` on the touched slots. Add a README "Examples" row. +- **Benchmark** `benches/event_pipeline.rs` (offline): decode throughput + (ERC-20 `Transfer`, V3 `Swap`, V3 `Mint`); `ingest_logs` per-block apply across + log-batch sizes (1 → 1000); `reorg_to` purge cost across touched-set sizes. + Register `[[bench]]` in `Cargo.toml`; mirror `benches/state_update.rs`; add a + README "Benchmarks" row. +- **CHANGELOG** `### Added`: the event pipeline (`EventDecoder`/`StateView`/ + `DecoderRegistry`/`EventPipeline`), the ERC-20 + V3 adapters, and the + `SlotMasked` vocabulary + `StateDiff.skipped_masks` (note the additive + `StateDiff` field + new `StateUpdate` variant under the pre-1.0 break policy). +- **ROADMAP**: flip the Phase 4 row to **Done** with the landing branch, a + "Landed on …" paragraph mirroring Phases 2/3. +- **KNOWN_ISSUES**: the §6.4 V3 fee-growth/oracle limitation. +- Rustdoc on **every** public item; module `//!` docs on `events` (Pillar B.2 + framing, `!Send` discipline, the events→Phase-3-vocabulary flow), `events/erc20`, + `events/uniswap_v3`. At least one runnable doctest (a pure decoder on a + hand-built `Log`, or the `SlotMasked` masked-write shape). + +## 12. Decisions (LOCKED) + +Confirmed with the user on 2026-06-16 before the acceptance tests were authored. + +**Decision 1 — packed-slot updates → `StateUpdate::SlotMasked`.** Add the +cold-aware RMW masked-write variant (§4.1) so a pure decoder can express a +partial update to a packed word (V3 `slot0`) without clobbering the bits it does +not own (notably `unlocked`). The "absolute clobber" and "impure decoder" +alternatives were rejected. + +**Decision 2 — V3 adapter coverage → `Swap` **and** `Mint`/`Burn` (full +ticks).** The adapter maintains `slot0`/`liquidity` from `Swap` and per-tick +`liquidityGross`/`liquidityNet`/`initialized` + `tickBitmap` + global +`liquidity` from `Mint`/`Burn` (§6). Fee-growth/oracle state is out of scope +(§6.4). Mint/Burn tick maintenance is computed against the `StateView` +(Decision 1's pure-data model needs the pre-state read). + +**Decision 3 — reorg → purge-and-resync touched addresses.** Track touched +addresses per block in a depth-bounded ring; `reorg_to(n)` purges everything +touched after `n` so reads re-fetch (§7.2). `ValidThrough` is the freshness +lever. Per-slot value rollback rejected. + +**Decision 4 — reconciliation → sampled re-read, correct **and** alarm.** +Opt-in `reconcile` samples event-derived slots and re-reads via `verify_slots`: +the fresh chain value wins (auto-correct) **and** the drift is surfaced (§7.3). +Honest freshness, built in from day one. Alarm-only and defer rejected. + +## 13. Build order (commit per step, green each time) + +1. `state_update.rs` + `cache/mod.rs`: `SlotMasked` variant, `slot_masked`, + `SkippedMask`, `StateDiff.skipped_masks` (+ merge/has_skipped/skipped_len), + serde, the apply arm (reuse `write_slot_through` + cold-aware read), and the + §10 masked-apply tests. Re-exports. +2. `events/mod.rs`: `StateView` (+ `impl … for EvmCache`), `EventDecoder`, + `DecoderRegistry` + dispatch tests. +3. `events/erc20.rs`: `Erc20TransferDecoder` + decoder/ingest tests. +4. `events/uniswap_v3.rs` (`protocols`): `UniswapV3Decoder`/`UniswapV3Layout`, + `Swap`/`Mint`/`Burn` + the §10 V3 tests. +5. `EventPipeline` (`ingest_logs`/`reorg_to`/`reconcile`/`derived_slots`) + + `BlockDigest`/`ReconcileReport`/`ReorgConfig` + the pipeline tests; the async + `drive`/`LogSource` convenience. +6. Example + benchmark + README rows. +7. Docs (module `//!`, item rustdoc, doctest), CHANGELOG, ROADMAP → Done, + KNOWN_ISSUES. + +## 14. Final acceptance + +Both feature configs green (§0). All new + existing tests pass. The example runs +offline and prints a non-trivial `BlockDigest`, a reorg purge, and a reconcile +alarm. The benchmark builds and runs (`cargo bench --no-run`). The V3 adapter +preserves the `slot0` `unlocked`/observation bits under `Swap` and maintains +tick gross/net/initialized/bitmap/global-liquidity under `Mint`/`Burn`, all +cold-aware. Report: what landed per file, the public API added, the decoder/ +adapter behavior (with the §6.4 limitation called out), test coverage, and the +verification output. From c3efe40bafa720a29538e77c2bb54faa590845c8 Mon Sep 17 00:00:00 2001 From: Kai Aldag Date: Tue, 16 Jun 2026 12:56:28 +0100 Subject: [PATCH 15/26] Phase 4: acceptance tests (red contract) for the event pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Authored before implementation, per the phased workflow — these define correctness for Pillar B.2 and will validate the deliverable: - tests/state_update.rs: SlotMasked cold-aware masked-write tests (sets only masked bits / no-op / cold skip+surface via skipped_masks / both-layer write-through / full-mask-vs-cold / serde round-trip). - tests/event_pipeline.rs: DecoderRegistry dispatch (address-scoped + global); ERC-20 Transfer -> Sub/Add SlotDeltas (mint/burn zero-address legs, per-token slot override, ingest conserves balances via real SLOAD, cold skip); pipeline reorg_to purge-and-resync + ReorgConfig scope; reconcile correct+alarm / match / no-fetcher error; UniswapV3 adapter (Swap preserves slot0 unlocked/observation bits + absolute liquidity + cold-skip; Mint gross/net signs + initialize + bitmap flip + in/out-of-range global liquidity; Burn uninitialize+clear via same-block sequencing; cold tick skip; unregistered pool no-op). Verified the alloy event-ABI plumbing (sol! Swap/Mint/Burn construction + encode_log_data + decode_log round-trip) in isolation before committing. Tests are RED until the Phase 4 surface lands. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/event_pipeline.rs | 821 ++++++++++++++++++++++++++++++++++++++++ tests/state_update.rs | 185 ++++++++- 2 files changed, 1004 insertions(+), 2 deletions(-) create mode 100644 tests/event_pipeline.rs diff --git a/tests/event_pipeline.rs b/tests/event_pipeline.rs new file mode 100644 index 0000000..f60653b --- /dev/null +++ b/tests/event_pipeline.rs @@ -0,0 +1,821 @@ +//! Offline acceptance tests for the Phase 4 event pipeline (Pillar B.2). +//! +//! These are the **contract** the implementation must satisfy: the +//! `EventDecoder` / `StateView` traits, the `DecoderRegistry`, the ERC-20 +//! `Transfer` decoder, the UniswapV3 `Swap`/`Mint`/`Burn` adapter, and the +//! `EventPipeline` (`ingest_logs` / `reorg_to` / `reconcile`). Everything runs +//! fully offline (mocked provider, state injected directly, logs built in +//! memory), so no test reaches the network. +//! +//! Layering vocabulary mirrors `tests/state_update.rs`: +//! - **layer 1 / overlay** = the CacheDB overlay (`db_mut().cache.accounts`). +//! - **layer 2 / backend** = the BlockchainDb backend (`blockchain_db()`). + +mod common; + +use std::collections::HashMap; +use std::sync::Arc; + +use alloy_primitives::{Address, Bytes, Log, U256, keccak256}; +use anyhow::Result; + +use common::{install_mock_erc20, setup_cache, stub_fetcher}; +use evm_fork_cache::cache::EvmCache; +use evm_fork_cache::events::{ + DecoderRegistry, EventDecoder, EventPipeline, ReorgConfig, StateView, +}; +use evm_fork_cache::events::erc20::Erc20TransferDecoder; +use evm_fork_cache::{PurgeScope, SlotDelta, StateUpdate}; + +// --------------------------------------------------------------------------- +// Shared helpers. +// --------------------------------------------------------------------------- + +/// Hashed storage slot of a `mapping(address => uint256)` at `mapping_slot`. +fn mapping_slot(owner: Address, mapping_slot: u64) -> U256 { + use alloy_sol_types::SolValue; + let key = keccak256((owner, U256::from(mapping_slot)).abi_encode()); + U256::from_be_bytes(key.0) +} + +/// Value of a slot in the BlockchainDb backend (layer 2) only. +fn backend_slot(cache: &EvmCache, addr: Address, slot: U256) -> Option { + cache + .blockchain_db() + .storage() + .read() + .get(&addr) + .and_then(|s| s.get(&slot).copied()) +} + +/// A read-only [`StateView`] stub backed by a fixed map (for pure decoder unit +/// tests that do not need a real cache). +struct StubView(HashMap<(Address, U256), U256>); +impl StateView for StubView { + fn storage(&self, address: Address, slot: U256) -> Option { + self.0.get(&(address, slot)).copied() + } +} +fn empty_view() -> StubView { + StubView(HashMap::new()) +} + +/// A test-only decoder that emits a single absolute `Slot` write for every log +/// (used to exercise pipeline mechanics independent of any real protocol). +struct MarkDecoder { + slot: U256, + value: U256, +} +impl EventDecoder for MarkDecoder { + fn decode(&self, log: &Log, _view: &dyn StateView) -> Vec { + vec![StateUpdate::slot(log.address, self.slot, self.value)] + } +} + +/// A test-only decoder that fires only for logs whose first topic equals `topic`. +struct TaggedDecoder { + topic: alloy_primitives::B256, + slot: U256, + value: U256, +} +impl EventDecoder for TaggedDecoder { + fn decode(&self, log: &Log, _view: &dyn StateView) -> Vec { + if log.topics().first() == Some(&self.topic) { + vec![StateUpdate::slot(log.address, self.slot, self.value)] + } else { + vec![] + } + } +} + +/// Build a bare log at `address` with the given topics and empty data. +fn bare_log(address: Address, topics: Vec) -> Log { + Log::new_unchecked(address, topics, Bytes::new()) +} + +// =========================================================================== +// EventDecoder / DecoderRegistry — dispatch. +// =========================================================================== + +#[test] +fn registry_dispatches_address_scoped_then_global() { + let token_a = Address::repeat_byte(0x0a); + let token_b = Address::repeat_byte(0x0b); + + let mut registry = DecoderRegistry::new(); + // Address-scoped: only fires for token_a logs. + registry.register_for_address( + token_a, + Arc::new(MarkDecoder { + slot: U256::from(1), + value: U256::from(11), + }), + ); + // Global: fires for every log. + registry.register(Arc::new(MarkDecoder { + slot: U256::from(9), + value: U256::from(99), + })); + + let view = empty_view(); + + // A token_a log hits both the scoped decoder and the global one. + let updates_a = registry.decode(&bare_log(token_a, vec![]), &view); + assert_eq!(updates_a.len(), 2); + assert!(updates_a.contains(&StateUpdate::slot(token_a, U256::from(1), U256::from(11)))); + assert!(updates_a.contains(&StateUpdate::slot(token_a, U256::from(9), U256::from(99)))); + + // A token_b log hits only the global decoder. + let updates_b = registry.decode(&bare_log(token_b, vec![]), &view); + assert_eq!( + updates_b, + vec![StateUpdate::slot(token_b, U256::from(9), U256::from(99))] + ); +} + +#[test] +fn decoder_returns_empty_for_unrecognized_log() { + let topic = keccak256(b"SomethingElse()"); + let decoder = TaggedDecoder { + topic: keccak256(b"Wanted()"), + slot: U256::from(0), + value: U256::from(1), + }; + let view = empty_view(); + let log = bare_log(Address::repeat_byte(0x01), vec![topic]); + assert!(decoder.decode(&log, &view).is_empty()); +} + +// =========================================================================== +// ERC-20 Transfer decoder. +// =========================================================================== + +/// Build an ERC-20 `Transfer(from, to, value)` log. +fn transfer_log(token: Address, from: Address, to: Address, value: U256) -> Log { + let sig = keccak256(b"Transfer(address,address,uint256)"); + let topics = vec![ + sig, + from.into_word(), + to.into_word(), + ]; + Log::new_unchecked(token, topics, Bytes::copy_from_slice(&value.to_be_bytes::<32>())) +} + +#[test] +fn erc20_transfer_decodes_to_sub_and_add_deltas() { + let token = Address::repeat_byte(0x20); + let from = Address::repeat_byte(0x21); + let to = Address::repeat_byte(0x22); + + let decoder = Erc20TransferDecoder::new(U256::from(3)); // default balance slot 3 + let view = empty_view(); + let updates = decoder.decode(&transfer_log(token, from, to, U256::from(100)), &view); + + assert_eq!( + updates, + vec![ + StateUpdate::slot_delta(token, mapping_slot(from, 3), SlotDelta::Sub(U256::from(100))), + StateUpdate::slot_delta(token, mapping_slot(to, 3), SlotDelta::Add(U256::from(100))), + ] + ); +} + +#[test] +fn erc20_mint_skips_zero_from_and_burn_skips_zero_to() { + let token = Address::repeat_byte(0x23); + let holder = Address::repeat_byte(0x24); + let decoder = Erc20TransferDecoder::new(U256::from(3)); + let view = empty_view(); + + // Mint: from == ZERO → only the Add leg. + let mint = decoder.decode( + &transfer_log(token, Address::ZERO, holder, U256::from(7)), + &view, + ); + assert_eq!( + mint, + vec![StateUpdate::slot_delta( + token, + mapping_slot(holder, 3), + SlotDelta::Add(U256::from(7)) + )] + ); + + // Burn: to == ZERO → only the Sub leg. + let burn = decoder.decode( + &transfer_log(token, holder, Address::ZERO, U256::from(7)), + &view, + ); + assert_eq!( + burn, + vec![StateUpdate::slot_delta( + token, + mapping_slot(holder, 3), + SlotDelta::Sub(U256::from(7)) + )] + ); +} + +#[test] +fn erc20_uses_per_token_slot_override_else_default() { + let token_default = Address::repeat_byte(0x25); + let token_custom = Address::repeat_byte(0x26); + let holder = Address::repeat_byte(0x27); + + let decoder = Erc20TransferDecoder::new(U256::from(3)).with_token(token_custom, U256::from(9)); + let view = empty_view(); + + let d = decoder.decode( + &transfer_log(token_default, Address::ZERO, holder, U256::from(1)), + &view, + ); + assert_eq!(d[0], StateUpdate::slot_delta(token_default, mapping_slot(holder, 3), SlotDelta::Add(U256::from(1)))); + + let c = decoder.decode( + &transfer_log(token_custom, Address::ZERO, holder, U256::from(1)), + &view, + ); + assert_eq!(c[0], StateUpdate::slot_delta(token_custom, mapping_slot(holder, 9), SlotDelta::Add(U256::from(1)))); +} + +#[test] +fn erc20_non_transfer_log_decodes_to_empty() { + let decoder = Erc20TransferDecoder::new(U256::from(3)); + let view = empty_view(); + // Wrong topic0. + let log = bare_log(Address::repeat_byte(0x28), vec![keccak256(b"Approval(address,address,uint256)")]); + assert!(decoder.decode(&log, &view).is_empty()); +} + +#[tokio::test] +async fn erc20_ingest_updates_balances_and_conserves() -> Result<()> { + use common::{balance_of, install_default_account}; + + let token = Address::repeat_byte(0x2a); + let alice = Address::repeat_byte(0x2b); + let bob = Address::repeat_byte(0x2c); + + let mut cache = setup_cache().await?; + install_default_account(&mut cache, Address::ZERO); + install_default_account(&mut cache, alice); + install_default_account(&mut cache, bob); + install_mock_erc20(&mut cache, token); + + // Seed both holders' balance slots (overlay-resident, EVM-visible). Balance + // mapping is slot 3 in the MockERC20 fixture. + cache + .db_mut() + .insert_account_storage(token, mapping_slot(alice, 3), U256::from(1000))?; + cache + .db_mut() + .insert_account_storage(token, mapping_slot(bob, 3), U256::from(500))?; + + let mut registry = DecoderRegistry::new(); + registry.register(Arc::new(Erc20TransferDecoder::new(U256::from(3)))); + let mut pipeline = EventPipeline::new(registry); + + // Alice transfers 200 to Bob. + let digest = pipeline.ingest_logs( + &mut cache, + 100, + &[transfer_log(token, alice, bob, U256::from(200))], + ); + + // Two slot changes applied; nothing skipped. + assert_eq!(digest.block, 100); + assert_eq!(digest.applied.slots.len(), 2); + assert!(!digest.applied.has_skipped()); + assert_eq!(digest.decoded_logs, 1); + + // Balances move by the delta — assert via real SLOAD (balanceOf). + assert_eq!(balance_of(&mut cache, token, alice)?, U256::from(800)); + assert_eq!(balance_of(&mut cache, token, bob)?, U256::from(700)); + // Conservation. + assert_eq!( + balance_of(&mut cache, token, alice)? + balance_of(&mut cache, token, bob)?, + U256::from(1500) + ); + Ok(()) +} + +#[tokio::test] +async fn erc20_cold_balance_transfer_is_skipped_and_surfaced() -> Result<()> { + // A normal forked account (NOT StorageCleared): an unseeded balance slot is + // cold, so the Sub/Add delta is skipped and surfaced. + let token = Address::repeat_byte(0x2d); + let alice = Address::repeat_byte(0x2e); + let bob = Address::repeat_byte(0x2f); + + let mut cache = setup_cache().await?; + + let mut registry = DecoderRegistry::new(); + registry.register(Arc::new(Erc20TransferDecoder::new(U256::from(3)))); + let mut pipeline = EventPipeline::new(registry); + + let digest = pipeline.ingest_logs( + &mut cache, + 1, + &[transfer_log(token, alice, bob, U256::from(10))], + ); + + // Both legs cold → no slot changes, two surfaced skips. + assert!(digest.applied.slots.is_empty()); + assert!(digest.applied.has_skipped()); + assert_eq!(digest.applied.skipped.len(), 2); + Ok(()) +} + +// =========================================================================== +// EventPipeline — reorg + reconcile (decoder-agnostic mechanics). +// =========================================================================== + +#[tokio::test] +async fn ingest_records_touched_slots() -> Result<()> { + let token = Address::repeat_byte(0x30); + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, token); + + let mut registry = DecoderRegistry::new(); + registry.register(Arc::new(MarkDecoder { + slot: U256::from(5), + value: U256::from(42), + })); + let mut pipeline = EventPipeline::new(registry); + + let digest = pipeline.ingest_logs(&mut cache, 10, &[bare_log(token, vec![])]); + assert!(digest.touched_slots.contains(&(token, U256::from(5)))); + assert_eq!(cache.cached_storage_value(token, U256::from(5)), Some(U256::from(42))); + Ok(()) +} + +#[tokio::test] +async fn reorg_to_purges_addresses_touched_after_head() -> Result<()> { + let token_a = Address::repeat_byte(0x31); // block 10 (survives) + let token_b = Address::repeat_byte(0x32); // block 11 (purged) + let token_c = Address::repeat_byte(0x33); // block 12 (purged) + let slot = U256::from(0); + + let mut cache = setup_cache().await?; + for t in [token_a, token_b, token_c] { + install_mock_erc20(&mut cache, t); + } + + let mut registry = DecoderRegistry::new(); + registry.register(Arc::new(MarkDecoder { + slot, + value: U256::from(77), + })); + let mut pipeline = EventPipeline::new(registry); + + pipeline.ingest_logs(&mut cache, 10, &[bare_log(token_a, vec![])]); + pipeline.ingest_logs(&mut cache, 11, &[bare_log(token_b, vec![])]); + pipeline.ingest_logs(&mut cache, 12, &[bare_log(token_c, vec![])]); + + // Everything written. + assert_eq!(backend_slot(&cache, token_a, slot), Some(U256::from(77))); + assert_eq!(backend_slot(&cache, token_b, slot), Some(U256::from(77))); + assert_eq!(backend_slot(&cache, token_c, slot), Some(U256::from(77))); + + // Reorg back to block 10: purge B and C (touched after 10), keep A. + let diff = pipeline.reorg_to(&mut cache, 10); + + assert_eq!(backend_slot(&cache, token_a, slot), Some(U256::from(77)), "A untouched"); + assert_eq!(backend_slot(&cache, token_b, slot), None, "B storage purged"); + assert_eq!(backend_slot(&cache, token_c, slot), None, "C storage purged"); + + // The returned diff records the purges (B and C only). + let purged: Vec
= diff.purged.iter().map(|r| r.address).collect(); + assert!(purged.contains(&token_b) && purged.contains(&token_c)); + assert!(!purged.contains(&token_a)); + Ok(()) +} + +#[tokio::test] +async fn reorg_config_account_scope_fully_drops_account() -> Result<()> { + let token = Address::repeat_byte(0x34); + let slot = U256::from(0); + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, token); + + let mut registry = DecoderRegistry::new(); + registry.register(Arc::new(MarkDecoder { + slot, + value: U256::from(5), + })); + let mut pipeline = EventPipeline::new(registry).with_reorg_config(ReorgConfig { + depth: 64, + scope: PurgeScope::Account, + }); + + pipeline.ingest_logs(&mut cache, 20, &[bare_log(token, vec![])]); + let diff = pipeline.reorg_to(&mut cache, 19); + + assert!(diff.purged.iter().any(|r| r.address == token && r.account_removed)); + Ok(()) +} + +#[tokio::test] +async fn reconcile_reports_mismatch_and_corrects() -> Result<()> { + let token = Address::repeat_byte(0x35); + let slot = U256::from(0); + + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, token); + + // Event pipeline writes an (incorrect) value 50. + let mut registry = DecoderRegistry::new(); + registry.register(Arc::new(MarkDecoder { + slot, + value: U256::from(50), + })); + let mut pipeline = EventPipeline::new(registry); + pipeline.ingest_logs(&mut cache, 1, &[bare_log(token, vec![])]); + assert_eq!(cache.cached_storage_value(token, slot), Some(U256::from(50))); + + // Chain truth is 100. Reconcile must surface the drift AND correct the cache. + cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([((token, slot), U256::from(100))]))); + let report = pipeline.reconcile(&mut cache, &[(token, slot)])?; + + assert_eq!(report.checked, 1); + assert_eq!(report.mismatched.len(), 1); + assert_eq!(report.mismatched[0].old, U256::from(50)); + assert_eq!(report.mismatched[0].new, U256::from(100)); + // Cache corrected to chain truth. + assert_eq!(cache.cached_storage_value(token, slot), Some(U256::from(100))); + Ok(()) +} + +#[tokio::test] +async fn reconcile_empty_when_event_state_matches_chain() -> Result<()> { + let token = Address::repeat_byte(0x36); + let slot = U256::from(0); + + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, token); + let mut registry = DecoderRegistry::new(); + registry.register(Arc::new(MarkDecoder { + slot, + value: U256::from(100), + })); + let mut pipeline = EventPipeline::new(registry); + pipeline.ingest_logs(&mut cache, 1, &[bare_log(token, vec![])]); + + // Chain agrees (100). + cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([((token, slot), U256::from(100))]))); + let report = pipeline.reconcile(&mut cache, &[(token, slot)])?; + assert!(report.mismatched.is_empty()); + Ok(()) +} + +#[tokio::test] +async fn reconcile_errors_without_fetcher() -> Result<()> { + let mut cache = setup_cache().await?; + let registry = DecoderRegistry::new(); + let mut pipeline = EventPipeline::new(registry); + let token = Address::repeat_byte(0x37); + assert!(pipeline.reconcile(&mut cache, &[(token, U256::from(0))]).is_err()); + Ok(()) +} + +// =========================================================================== +// UniswapV3 adapter (protocols-gated). +// =========================================================================== + +#[cfg(feature = "protocols")] +mod uniswap_v3 { + use super::*; + use alloy_primitives::aliases::{I24, U160}; + use alloy_primitives::I256; + use alloy_sol_types::{SolEvent, sol}; + use evm_fork_cache::cache::{ + V3_LIQUIDITY_SLOT, V3_SLOT0_SLOT, V3_TICKS_BASE_SLOT, V3_TICK_BITMAP_BASE_SLOT, + v3_tick_bitmap_storage_key_with_base, v3_tick_info_storage_keys_with_base, + }; + use evm_fork_cache::events::uniswap_v3::{UniswapV3Decoder, UniswapV3Layout}; + + sol! { + event Swap(address indexed sender, address indexed recipient, int256 amount0, int256 amount1, uint160 sqrtPriceX96, uint128 liquidity, int24 tick); + event Mint(address sender, address indexed owner, int24 indexed tickLower, int24 indexed tickUpper, uint128 amount, uint256 amount0, uint256 amount1); + event Burn(address indexed owner, int24 indexed tickLower, int24 indexed tickUpper, uint128 amount, uint256 amount0, uint256 amount1); + } + + fn swap_log(pool: Address, sqrt_price: u128, liquidity: u128, tick: i32) -> Log { + let ev = Swap { + sender: Address::repeat_byte(0x01), + recipient: Address::repeat_byte(0x02), + amount0: I256::try_from(-1i64).unwrap(), + amount1: I256::try_from(1i64).unwrap(), + sqrtPriceX96: U160::from(sqrt_price), + liquidity, + tick: I24::try_from(tick).unwrap(), + }; + Log { + address: pool, + data: ev.encode_log_data(), + } + } + + fn mint_log(pool: Address, tick_lower: i32, tick_upper: i32, amount: u128) -> Log { + let ev = Mint { + sender: Address::repeat_byte(0x03), + owner: Address::repeat_byte(0x04), + tickLower: I24::try_from(tick_lower).unwrap(), + tickUpper: I24::try_from(tick_upper).unwrap(), + amount, + amount0: U256::from(1), + amount1: U256::from(1), + }; + Log { + address: pool, + data: ev.encode_log_data(), + } + } + + fn burn_log(pool: Address, tick_lower: i32, tick_upper: i32, amount: u128) -> Log { + let ev = Burn { + owner: Address::repeat_byte(0x04), + tickLower: I24::try_from(tick_lower).unwrap(), + tickUpper: I24::try_from(tick_upper).unwrap(), + amount, + amount0: U256::from(1), + amount1: U256::from(1), + }; + Log { + address: pool, + data: ev.encode_log_data(), + } + } + + /// Pack a slot0 word: sqrtPriceX96 [0,160), tick [160,184) (int24), and an + /// arbitrary `high` block of preserved bits at [184,256). + fn pack_slot0(sqrt_price: u128, tick: i32, high: U256) -> U256 { + let tick24 = U256::from((tick as u32) & 0x00FF_FFFF); + U256::from(sqrt_price) | (tick24 << 160) | (high << 184) + } + + fn tick_word(gross: u128, net: i128) -> U256 { + U256::from(gross) | (U256::from(net as u128) << 128) + } + fn unpack_tick_word(w: U256) -> (u128, i128) { + let gross = u128::try_from(w & U256::from(u128::MAX)).unwrap(); + let net = u128::try_from((w >> 128) & U256::from(u128::MAX)).unwrap() as i128; + (gross, net) + } + + fn pool_with_decoder(tick_spacing: i32) -> (Address, UniswapV3Decoder) { + let pool = Address::repeat_byte(0x40); + let decoder = UniswapV3Decoder::new().with_pool(pool, UniswapV3Layout::uniswap(tick_spacing)); + (pool, decoder) + } + + #[tokio::test] + async fn v3_swap_sets_price_and_tick_preserving_unlocked() -> Result<()> { + let (pool, decoder) = pool_with_decoder(1); + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, pool); + + // Seed slot0 with old price/tick AND the unlocked bit (240) + a nonzero + // observation index (bits 184+). high = unlocked(bit 56 of high) | obs(=7). + let high = (U256::from(1) << 56) | U256::from(7); + let seeded = pack_slot0(1_000_000, 50, high); + cache.db_mut().insert_account_storage(pool, V3_SLOT0_SLOT, seeded)?; + + let mut registry = DecoderRegistry::new(); + registry.register(Arc::new(decoder)); + let mut pipeline = EventPipeline::new(registry); + + pipeline.ingest_logs(&mut cache, 1, &[swap_log(pool, 2_000_000, 9999, 75)]); + + let result = cache.cached_storage_value(pool, V3_SLOT0_SLOT).unwrap(); + // Low 184 bits are the new price + tick. + let low_mask = (U256::from(1) << 184) - U256::from(1); + let expected_low = U256::from(2_000_000u64) | (U256::from(75u64) << 160); + assert_eq!(result & low_mask, expected_low, "price+tick updated"); + // High bits (incl. unlocked) preserved. + assert_eq!(result >> 184, high, "observation/unlocked bits preserved"); + Ok(()) + } + + #[tokio::test] + async fn v3_swap_sets_liquidity_absolute() -> Result<()> { + let (pool, decoder) = pool_with_decoder(1); + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, pool); + cache + .db_mut() + .insert_account_storage(pool, V3_SLOT0_SLOT, pack_slot0(1, 0, U256::from(1) << 56))?; + + let mut registry = DecoderRegistry::new(); + registry.register(Arc::new(decoder)); + let mut pipeline = EventPipeline::new(registry); + + pipeline.ingest_logs(&mut cache, 1, &[swap_log(pool, 1, 123_456, 0)]); + assert_eq!( + cache.cached_storage_value(pool, V3_LIQUIDITY_SLOT), + Some(U256::from(123_456u64)) + ); + Ok(()) + } + + #[tokio::test] + async fn v3_swap_cold_slot0_is_skipped() -> Result<()> { + // Fresh pool with no account → slot0 is cold. + let pool = Address::repeat_byte(0x41); + let decoder = UniswapV3Decoder::new().with_pool(pool, UniswapV3Layout::uniswap(1)); + let mut cache = setup_cache().await?; + + let mut registry = DecoderRegistry::new(); + registry.register(Arc::new(decoder)); + let mut pipeline = EventPipeline::new(registry); + + let digest = pipeline.ingest_logs(&mut cache, 1, &[swap_log(pool, 5, 5, 5)]); + // slot0 masked write skipped (un-masked bits unknown). + assert!(digest.applied.has_skipped()); + assert_eq!(cache.cached_storage_value(pool, V3_SLOT0_SLOT), None); + Ok(()) + } + + #[tokio::test] + async fn v3_mint_increments_gross_and_net_with_correct_signs() -> Result<()> { + let (pool, decoder) = pool_with_decoder(1); + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, pool); // StorageCleared → unseeded ticks read 0 (hot) + + let mut registry = DecoderRegistry::new(); + registry.register(Arc::new(decoder)); + let mut pipeline = EventPipeline::new(registry); + + let (lo, hi, amount) = (10i32, 20i32, 500u128); + pipeline.ingest_logs(&mut cache, 1, &[mint_log(pool, lo, hi, amount)]); + + let lo_key = v3_tick_info_storage_keys_with_base(lo, V3_TICKS_BASE_SLOT)[0]; + let hi_key = v3_tick_info_storage_keys_with_base(hi, V3_TICKS_BASE_SLOT)[0]; + let (lo_gross, lo_net) = unpack_tick_word(cache.cached_storage_value(pool, lo_key).unwrap()); + let (hi_gross, hi_net) = unpack_tick_word(cache.cached_storage_value(pool, hi_key).unwrap()); + + assert_eq!(lo_gross, 500); + assert_eq!(lo_net, 500, "lower tick: net += amount"); + assert_eq!(hi_gross, 500); + assert_eq!(hi_net, -500, "upper tick: net -= amount"); + Ok(()) + } + + #[tokio::test] + async fn v3_mint_initializes_tick_and_flips_bitmap() -> Result<()> { + let (pool, decoder) = pool_with_decoder(1); + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, pool); + + let mut registry = DecoderRegistry::new(); + registry.register(Arc::new(decoder)); + let mut pipeline = EventPipeline::new(registry); + + let (lo, hi) = (10i32, 20i32); + pipeline.ingest_logs(&mut cache, 1, &[mint_log(pool, lo, hi, 500)]); + + // initialized flag (slot +3, bit 248) set for both ticks. + let lo3 = v3_tick_info_storage_keys_with_base(lo, V3_TICKS_BASE_SLOT)[3]; + let hi3 = v3_tick_info_storage_keys_with_base(hi, V3_TICKS_BASE_SLOT)[3]; + let init_bit = U256::from(1) << 248; + assert_eq!(cache.cached_storage_value(pool, lo3).unwrap() & init_bit, init_bit); + assert_eq!(cache.cached_storage_value(pool, hi3).unwrap() & init_bit, init_bit); + + // bitmap word 0 (ticks 10 & 20 with tick_spacing 1 → word 0, bits 10 & 20). + let word_key = v3_tick_bitmap_storage_key_with_base(0, V3_TICK_BITMAP_BASE_SLOT); + let bitmap = cache.cached_storage_value(pool, word_key).unwrap(); + assert_eq!(bitmap & (U256::from(1) << 10), U256::from(1) << 10); + assert_eq!(bitmap & (U256::from(1) << 20), U256::from(1) << 20); + Ok(()) + } + + #[tokio::test] + async fn v3_burn_to_zero_uninitializes_and_clears_bitmap() -> Result<()> { + let (pool, decoder) = pool_with_decoder(1); + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, pool); + + let mut registry = DecoderRegistry::new(); + registry.register(Arc::new(decoder)); + let mut pipeline = EventPipeline::new(registry); + + let (lo, hi) = (10i32, 20i32); + // Same-block Mint then Burn of the full amount: the Burn decode sees the + // Mint's applied gross/net via the StateView, returning the tick to 0. + pipeline.ingest_logs( + &mut cache, + 1, + &[mint_log(pool, lo, hi, 500), burn_log(pool, lo, hi, 500)], + ); + + let lo_key = v3_tick_info_storage_keys_with_base(lo, V3_TICKS_BASE_SLOT)[0]; + let (lo_gross, lo_net) = unpack_tick_word(cache.cached_storage_value(pool, lo_key).unwrap()); + assert_eq!(lo_gross, 0, "gross back to zero"); + assert_eq!(lo_net, 0, "net back to zero"); + + // initialized flag cleared and bitmap bit cleared. + let lo3 = v3_tick_info_storage_keys_with_base(lo, V3_TICKS_BASE_SLOT)[3]; + let init_bit = U256::from(1) << 248; + assert_eq!(cache.cached_storage_value(pool, lo3).unwrap_or(U256::ZERO) & init_bit, U256::ZERO); + let word_key = v3_tick_bitmap_storage_key_with_base(0, V3_TICK_BITMAP_BASE_SLOT); + assert_eq!( + cache.cached_storage_value(pool, word_key).unwrap_or(U256::ZERO) & (U256::from(1) << 10), + U256::ZERO + ); + Ok(()) + } + + #[tokio::test] + async fn v3_mint_updates_global_liquidity_when_in_range() -> Result<()> { + let (pool, decoder) = pool_with_decoder(1); + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, pool); + + // Current tick 15 is within [10, 20); liquidity seeded to 1000. + cache.db_mut().insert_account_storage( + pool, + V3_SLOT0_SLOT, + pack_slot0(1_000_000, 15, U256::from(1) << 56), + )?; + cache + .db_mut() + .insert_account_storage(pool, V3_LIQUIDITY_SLOT, U256::from(1000))?; + + let mut registry = DecoderRegistry::new(); + registry.register(Arc::new(decoder)); + let mut pipeline = EventPipeline::new(registry); + + pipeline.ingest_logs(&mut cache, 1, &[mint_log(pool, 10, 20, 500)]); + assert_eq!( + cache.cached_storage_value(pool, V3_LIQUIDITY_SLOT), + Some(U256::from(1500)), + "in-range mint adds to global liquidity" + ); + Ok(()) + } + + #[tokio::test] + async fn v3_mint_leaves_global_liquidity_when_out_of_range() -> Result<()> { + let (pool, decoder) = pool_with_decoder(1); + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, pool); + + // Current tick 5 is BELOW [10, 20); liquidity must not change. + cache.db_mut().insert_account_storage( + pool, + V3_SLOT0_SLOT, + pack_slot0(1_000_000, 5, U256::from(1) << 56), + )?; + cache + .db_mut() + .insert_account_storage(pool, V3_LIQUIDITY_SLOT, U256::from(1000))?; + + let mut registry = DecoderRegistry::new(); + registry.register(Arc::new(decoder)); + let mut pipeline = EventPipeline::new(registry); + + pipeline.ingest_logs(&mut cache, 1, &[mint_log(pool, 10, 20, 500)]); + assert_eq!( + cache.cached_storage_value(pool, V3_LIQUIDITY_SLOT), + Some(U256::from(1000)), + "out-of-range mint does not touch global liquidity" + ); + Ok(()) + } + + #[tokio::test] + async fn v3_mint_cold_tick_word_is_skipped() -> Result<()> { + // Fresh pool, no account → tick words are cold (None), so the tick + // maintenance is skipped and surfaced rather than computed against 0. + let pool = Address::repeat_byte(0x42); + let decoder = UniswapV3Decoder::new().with_pool(pool, UniswapV3Layout::uniswap(1)); + let mut cache = setup_cache().await?; + + let mut registry = DecoderRegistry::new(); + registry.register(Arc::new(decoder)); + let mut pipeline = EventPipeline::new(registry); + + let digest = pipeline.ingest_logs(&mut cache, 1, &[mint_log(pool, 10, 20, 500)]); + assert!(digest.applied.has_skipped(), "cold tick words surfaced as skips"); + let lo_key = v3_tick_info_storage_keys_with_base(10, V3_TICKS_BASE_SLOT)[0]; + assert_eq!(cache.cached_storage_value(pool, lo_key), None, "nothing written"); + Ok(()) + } + + #[tokio::test] + async fn v3_unregistered_pool_decodes_to_nothing() -> Result<()> { + let known = Address::repeat_byte(0x43); + let unknown = Address::repeat_byte(0x44); + let decoder = UniswapV3Decoder::new().with_pool(known, UniswapV3Layout::uniswap(1)); + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, unknown); + + let mut registry = DecoderRegistry::new(); + registry.register(Arc::new(decoder)); + let mut pipeline = EventPipeline::new(registry); + + let digest = pipeline.ingest_logs(&mut cache, 1, &[swap_log(unknown, 5, 5, 5)]); + assert!(digest.applied.is_empty() && !digest.applied.has_skipped()); + assert_eq!(digest.decoded_logs, 0); + Ok(()) + } +} diff --git a/tests/state_update.rs b/tests/state_update.rs index 825cc31..3647b8e 100644 --- a/tests/state_update.rs +++ b/tests/state_update.rs @@ -22,8 +22,8 @@ use common::{ }; use evm_fork_cache::cache::EvmCache; use evm_fork_cache::{ - AccountPatch, PurgeScope, SkippedBalanceDelta, SkippedDelta, SlotChange, SlotDelta, StateDiff, - StateUpdate, + AccountPatch, PurgeScope, SkippedBalanceDelta, SkippedDelta, SkippedMask, SlotChange, SlotDelta, + StateDiff, StateUpdate, }; use revm::state::{AccountInfo, Bytecode}; @@ -1606,3 +1606,184 @@ async fn account_patch_normalizes_zero_code_hash_across_layers() -> Result<()> { ); Ok(()) } + +// --------------------------------------------------------------------------- +// Phase 4 — SlotMasked: cold-aware read-modify-write masked slot write. +// +// `new = (old & !mask) | (value & mask)`. Only the `mask` bits are touched; the +// rest of the packed word is preserved. Cold (slot absent from both layers) is +// skipped and surfaced in `diff.skipped_masks` (the un-masked bits are unknown). +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn slot_masked_sets_only_masked_bits() -> Result<()> { + let token = Address::repeat_byte(0x11); + let slot = U256::from(0); + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, token); + // Seed a packed word with the high bits set and the low byte clear. + let seeded = U256::MAX - U256::from(0xFF); // 0xFF..FF00 + cache.db_mut().insert_account_storage(token, slot, seeded)?; + + // Mask the low byte only, set it to 0x42. + let diff = cache.apply_update(&StateUpdate::slot_masked( + token, + slot, + U256::from(0xFF), + U256::from(0x42), + )); + + let expected = seeded | U256::from(0x42); // high bits preserved, low byte = 0x42 + assert_eq!(cache.cached_storage_value(token, slot), Some(expected)); + assert_eq!( + diff.slots, + vec![SlotChange { + address: token, + slot, + old: seeded, + new: expected, + }] + ); + assert!(diff.skipped_masks.is_empty()); + Ok(()) +} + +#[tokio::test] +async fn slot_masked_noop_when_masked_bits_already_equal() -> Result<()> { + let token = Address::repeat_byte(0x12); + let slot = U256::from(1); + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, token); + cache + .db_mut() + .insert_account_storage(token, slot, U256::from(0x42))?; + + // The masked bits already equal the target → no change. + let diff = cache.apply_update(&StateUpdate::slot_masked( + token, + slot, + U256::from(0xFF), + U256::from(0x42), + )); + + assert!(diff.is_empty(), "masked write that changes nothing is a no-op"); + assert!(diff.skipped_masks.is_empty()); + Ok(()) +} + +#[tokio::test] +async fn slot_masked_cold_slot_is_skipped_and_surfaced() -> Result<()> { + // Fresh address with no overlay account and no backend value: the slot is + // cold. A masked write cannot know the un-masked bits, so it is skipped. + let pool = Address::repeat_byte(0x13); + let slot = U256::from(0); + let mut cache = setup_cache().await?; + + let diff = cache.apply_update(&StateUpdate::slot_masked( + pool, + slot, + U256::from(0xFF), + U256::from(0x42), + )); + + assert!(diff.slots.is_empty()); + assert_eq!( + diff.skipped_masks, + vec![SkippedMask { + address: pool, + slot, + mask: U256::from(0xFF), + value: U256::from(0x42), + }] + ); + assert!(diff.has_skipped()); + assert!(!diff.is_fully_applied()); + assert_eq!(diff.skipped_len(), 1); + // Still cold — nothing was written. + assert_eq!(cache.cached_storage_value(pool, slot), None); + Ok(()) +} + +#[tokio::test] +async fn slot_masked_writes_through_both_layers() -> Result<()> { + let token = Address::repeat_byte(0x14); + let slot = U256::from(2); + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, token); + // Overlay-resident (hot) seed so an overlay account exists. + let seeded = U256::from(0xFF00); + cache.db_mut().insert_account_storage(token, slot, seeded)?; + + cache.apply_update(&StateUpdate::slot_masked( + token, + slot, + U256::from(0x00FF), + U256::from(0x0042), + )); + + let expected = U256::from(0xFF42); + assert_eq!( + overlay_slot(&mut cache, token, slot), + Some(expected), + "overlay (layer 1) updated" + ); + assert_eq!( + backend_slot(&cache, token, slot), + Some(expected), + "backend (layer 2) updated" + ); + Ok(()) +} + +#[tokio::test] +async fn slot_masked_full_mask_equals_absolute_on_hot_but_skips_cold() -> Result<()> { + // mask == U256::MAX behaves like an absolute write on a hot slot, but still + // skip-and-surfaces on a cold one (unlike StateUpdate::Slot). + let token = Address::repeat_byte(0x15); + let hot = U256::from(0); + let cold_addr = Address::repeat_byte(0x16); + let cold = U256::from(0); + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, token); + cache + .db_mut() + .insert_account_storage(token, hot, U256::from(7))?; + + let hot_diff = + cache.apply_update(&StateUpdate::slot_masked(token, hot, U256::MAX, U256::from(99))); + assert_eq!(cache.cached_storage_value(token, hot), Some(U256::from(99))); + assert_eq!(hot_diff.slots.len(), 1); + assert!(hot_diff.skipped_masks.is_empty()); + + let cold_diff = cache.apply_update(&StateUpdate::slot_masked( + cold_addr, + cold, + U256::MAX, + U256::from(99), + )); + assert!(cold_diff.slots.is_empty()); + assert_eq!(cold_diff.skipped_masks.len(), 1); + assert_eq!(cache.cached_storage_value(cold_addr, cold), None); + Ok(()) +} + +#[tokio::test] +async fn slot_masked_serde_round_trips() -> Result<()> { + let update = + StateUpdate::slot_masked(Address::repeat_byte(0x17), U256::from(5), U256::from(0xFF), U256::from(3)); + let json = serde_json::to_string(&update)?; + let back: StateUpdate = serde_json::from_str(&json)?; + assert_eq!(update, back); + + let mut diff = StateDiff::default(); + diff.skipped_masks.push(SkippedMask { + address: Address::repeat_byte(0x18), + slot: U256::from(1), + mask: U256::from(0xFF), + value: U256::from(2), + }); + let json = serde_json::to_string(&diff)?; + let back: StateDiff = serde_json::from_str(&json)?; + assert_eq!(diff, back); + Ok(()) +} From 52d956bb748dc0d0dc08f44d1b1effd7048d9003 Mon Sep 17 00:00:00 2001 From: Kai Aldag Date: Tue, 16 Jun 2026 13:02:04 +0100 Subject: [PATCH 16/26] Phase 4 step 1: SlotMasked cold-aware masked-write vocabulary Add StateUpdate::SlotMasked (new(old & !mask) | (value & mask)), the slot_masked constructor, the SkippedMask leaf record, and the StateDiff.skipped_masks field (+ merge / has_skipped / skipped_len). Apply arm in cache/mod.rs mirrors the SlotDelta arm: cold-aware RMW via write_slot_through. Re-export SkippedMask. Makes the tests/state_update.rs SlotMasked block green. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/cache/mod.rs | 32 +++++++- src/lib.rs | 2 +- src/state_update.rs | 159 ++++++++++++++++++++++++++++++++++++++-- tests/event_pipeline.rs | 157 +++++++++++++++++++++++++++++---------- tests/state_update.rs | 25 +++++-- 5 files changed, 324 insertions(+), 51 deletions(-) diff --git a/src/cache/mod.rs b/src/cache/mod.rs index 1b4a458..b20f387 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -68,7 +68,7 @@ use crate::freshness::SlotChange; use crate::inspector::TransferInspector; use crate::state_update::{ AccountChange, AccountPatch, PurgeRecord, PurgeScope, SkippedBalanceDelta, SkippedDelta, - SlotDelta, StateDiff, StateUpdate, + SkippedMask, SlotDelta, StateDiff, StateUpdate, }; use bytecode::BytecodeCache; @@ -1264,6 +1264,36 @@ impl EvmCache { delta: *delta, }), }, + StateUpdate::SlotMasked { + address, + slot, + mask, + value, + } => match self.cached_storage_value(*address, *slot) { + // Hot slot: overwrite only the masked bits, preserving the rest. + // Build the change from the value we already read (mirroring the + // `SlotDelta` arm; do not re-read through `apply_slot`). + Some(old) => { + let new = (old & !*mask) | (*value & *mask); + self.write_slot_through(*address, *slot, new); + if old != new { + diff.slots.push(SlotChange { + address: *address, + slot: *slot, + old, + new, + }); + } + } + // Cold slot: the un-masked bits are unknown, so the result cannot + // be computed; write nothing and surface the skip for re-seeding. + None => diff.skipped_masks.push(SkippedMask { + address: *address, + slot: *slot, + mask: *mask, + value: *value, + }), + }, StateUpdate::BalanceDelta { address, delta } => { match self.apply_balance_delta(*address, *delta) { // Hot account: the saturating delta was applied. diff --git a/src/lib.rs b/src/lib.rs index 7428463..bc20262 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -116,5 +116,5 @@ pub use freshness::{ }; pub use state_update::{ AccountChange, AccountPatch, PurgeRecord, PurgeScope, SkippedBalanceDelta, SkippedDelta, - SlotDelta, StateDiff, StateUpdate, + SkippedMask, SlotDelta, StateDiff, StateUpdate, }; diff --git a/src/state_update.rs b/src/state_update.rs index b448107..165bff5 100644 --- a/src/state_update.rs +++ b/src/state_update.rs @@ -58,6 +58,20 @@ //! true value (the next read otherwise lazily fetches it). `modify_slot` hands its //! closure an `Option` (`None` when cold) and lets the caller decide. //! +//! # Masked writes to packed words +//! +//! A storage slot often packs several fields (a UniswapV3 `slot0` holds +//! `sqrtPriceX96`, `tick`, an observation index, and the `unlocked` flag in one +//! word). A decoder that learns only some of those fields must update *just* its +//! bits without clobbering the rest. [`StateUpdate::SlotMasked`] is the +//! cold-aware masked read-modify-write for exactly this: it computes +//! `new = (old & !mask) | (value & mask)`, touching only the `mask` bits. Like +//! [`SlotDelta`](StateUpdate::SlotDelta) it is cold-aware — the un-masked bits of +//! a slot absent from both layers are unknown, so a masked write to a cold slot +//! is **not** applied; it is surfaced in [`StateDiff::skipped_masks`] as a +//! [`SkippedMask`]. (A full-mask `SlotMasked` matches an absolute +//! [`Slot`](StateUpdate::Slot) write on a hot slot but still skips on a cold one.) +//! //! The same relative, cold-aware rule extends to an account's **native balance**: //! [`StateUpdate::BalanceDelta`] (and the closure form //! [`EvmCache::modify_account_balance`](crate::cache::EvmCache::modify_account_balance)) @@ -206,6 +220,32 @@ pub enum StateUpdate { /// The relative, saturating mutation to apply to the current balance. delta: SlotDelta, }, + /// Set only the `mask` bits of a storage slot to the corresponding bits of + /// `value`, preserving the rest: `new = (old & !mask) | (value & mask)`. + /// + /// A *masked* read-modify-write: it lets a pure decoder express a partial + /// update to a **packed** storage word (e.g. a UniswapV3 `slot0`, which packs + /// `sqrtPriceX96`, `tick`, the observation index, and the `unlocked` flag into + /// one slot) without knowing or clobbering the bits it does not own. + /// + /// **Cold-aware** — a masked write to a slot absent from *both* cache layers is + /// **not** applied (the un-masked bits are unknown, so the result cannot be + /// computed); it is surfaced in [`StateDiff::skipped_masks`] as a + /// [`SkippedMask`] instead. A masked write with `mask == U256::MAX` equals an + /// absolute [`Slot`](Self::Slot) write on a *hot* slot, but **still skips** on a + /// cold one (unlike [`Slot`](Self::Slot), which writes unconditionally). Use + /// [`Slot`](Self::Slot) for an unconditional absolute write; use `SlotMasked` + /// when neighbouring bits must be preserved. + SlotMasked { + /// Contract whose storage is written. + address: Address, + /// Storage slot key. + slot: U256, + /// Which bits of the slot to overwrite (1 = take from `value`). + mask: U256, + /// The bits to write (only the bits selected by `mask` are applied). + value: U256, + }, /// Patch an account's balance/nonce/code (partial — see [`AccountPatch`]). /// /// # Warning @@ -250,6 +290,17 @@ impl StateUpdate { } } + /// Construct a [`StateUpdate::SlotMasked`] that sets only the `mask` bits of + /// `(address, slot)` to the corresponding bits of `value`. + pub fn slot_masked(address: Address, slot: U256, mask: U256, value: U256) -> Self { + Self::SlotMasked { + address, + slot, + mask, + value, + } + } + /// Construct a [`StateUpdate::BalanceDelta`] that applies `delta` relative to /// the account's current native balance. pub fn balance_delta(address: Address, delta: SlotDelta) -> Self { @@ -428,6 +479,11 @@ pub struct StateDiff { /// was unknown). Like [`skipped`](Self::skipped) this is informational /// metadata, not a change. pub skipped_balances: Vec, + /// Masked slot updates ([`StateUpdate::SlotMasked`]) that were **not** applied + /// because the target slot's current value was unknown (cold) — the un-masked + /// bits could not be preserved. Like [`skipped`](Self::skipped) this is + /// informational metadata, not a change. + pub skipped_masks: Vec, } impl StateDiff { @@ -456,12 +512,15 @@ impl StateDiff { /// [`is_empty`](Self::is_empty) — callers applying relative updates should /// check this to avoid silently dropping a balance update. pub fn has_skipped(&self) -> bool { - !self.skipped.is_empty() || !self.skipped_balances.is_empty() + !self.skipped.is_empty() + || !self.skipped_balances.is_empty() + || !self.skipped_masks.is_empty() } - /// Total number of skipped relative updates (`skipped` + `skipped_balances`). + /// Total number of skipped relative/masked updates (`skipped` + + /// `skipped_balances` + `skipped_masks`). pub fn skipped_len(&self) -> usize { - self.skipped.len() + self.skipped_balances.len() + self.skipped.len() + self.skipped_balances.len() + self.skipped_masks.len() } /// Whether every relative update in the apply was applied (none skipped). @@ -475,14 +534,15 @@ impl StateDiff { /// /// Used by [`apply_updates`](crate::cache::EvmCache::apply_updates) to merge /// per-update diffs; the concatenation preserves order, so two writes to the - /// same slot record their `old → new` history in sequence. The `skipped` and - /// `skipped_balances` metadata are concatenated too. + /// same slot record their `old → new` history in sequence. The `skipped`, + /// `skipped_balances`, and `skipped_masks` metadata are concatenated too. pub fn merge(&mut self, other: StateDiff) { self.slots.extend(other.slots); self.accounts.extend(other.accounts); self.purged.extend(other.purged); self.skipped.extend(other.skipped); self.skipped_balances.extend(other.skipped_balances); + self.skipped_masks.extend(other.skipped_masks); } } @@ -549,6 +609,39 @@ pub struct SkippedBalanceDelta { pub delta: SlotDelta, } +/// A masked write ([`StateUpdate::SlotMasked`]) that could not be applied because +/// the target slot's current value is unknown (not cached in either layer). +/// +/// A masked write needs the slot's current value to preserve the un-masked bits +/// (`new = (old & !mask) | (value & mask)`); without it the result cannot be +/// computed, so the write is skipped rather than applied against an assumed value. +/// It is surfaced here so the caller can fetch+seed the slot and retry; otherwise +/// the next read lazily fetches the true value. +/// +/// # The `mask == U256::MAX, value == U256::ZERO` cold-tick convention +/// +/// The UniswapV3 adapter (`events::uniswap_v3`) reuses this record as a +/// "could-not-compute" marker for the *absolute* tick-word / global-liquidity +/// writes it must skip when a needed word is cold: it pushes a `SkippedMask` with +/// `mask == U256::MAX` and `value == U256::ZERO`. This avoids adding a fourth skip +/// vector for stateful protocol updates; the count flows through +/// [`StateDiff::skipped_len`] all the same, and the caller re-seeds the pool. +/// +/// Deliberately **not** `#[non_exhaustive]`: it is a stable, fully-determined leaf +/// record routinely constructed as a struct literal in equality assertions by the +/// test suite and downstream users testing against a returned diff. +#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct SkippedMask { + /// Contract whose storage slot the masked write targeted. + pub address: Address, + /// Storage slot key that was cold. + pub slot: U256, + /// The mask that was not applied. + pub mask: U256, + /// The value bits that were not applied. + pub value: U256, +} + #[cfg(test)] mod tests { use super::*; @@ -704,4 +797,60 @@ mod tests { assert!(left.is_empty()); assert_eq!(left.len(), 0); } + + #[test] + fn slot_masked_constructor_produces_variant() { + let a = addr(0xee); + assert_eq!( + StateUpdate::slot_masked(a, U256::from(1), U256::from(0xFF), U256::from(0x42)), + StateUpdate::SlotMasked { + address: a, + slot: U256::from(1), + mask: U256::from(0xFF), + value: U256::from(0x42), + } + ); + } + + #[test] + fn state_diff_merge_extends_skipped_masks_without_counting_it() { + let a = addr(0xef); + let mut left = StateDiff::default(); + let mut right = StateDiff::default(); + right.skipped_masks.push(SkippedMask { + address: a, + slot: U256::from(1), + mask: U256::from(0xFF), + value: U256::from(0x42), + }); + + left.merge(right); + assert_eq!(left.skipped_masks.len(), 1); + // A masked skip is metadata, not a change. + assert!(left.is_empty()); + assert_eq!(left.len(), 0); + // But it is discoverable through the skip accessors. + assert!(left.has_skipped()); + assert_eq!(left.skipped_len(), 1); + assert!(!left.is_fully_applied()); + } + + #[test] + fn slot_masked_serde_round_trips() { + let a = addr(0xf0); + let update = StateUpdate::slot_masked(a, U256::from(5), U256::from(0xFF), U256::from(3)); + let json = serde_json::to_string(&update).expect("serialize"); + let back: StateUpdate = serde_json::from_str(&json).expect("deserialize"); + assert_eq!(update, back); + + let mask = SkippedMask { + address: a, + slot: U256::from(1), + mask: U256::from(0xFF), + value: U256::from(2), + }; + let json = serde_json::to_string(&mask).expect("serialize"); + let back: SkippedMask = serde_json::from_str(&json).expect("deserialize"); + assert_eq!(mask, back); + } } diff --git a/tests/event_pipeline.rs b/tests/event_pipeline.rs index f60653b..bef20d7 100644 --- a/tests/event_pipeline.rs +++ b/tests/event_pipeline.rs @@ -21,10 +21,10 @@ use anyhow::Result; use common::{install_mock_erc20, setup_cache, stub_fetcher}; use evm_fork_cache::cache::EvmCache; +use evm_fork_cache::events::erc20::Erc20TransferDecoder; use evm_fork_cache::events::{ DecoderRegistry, EventDecoder, EventPipeline, ReorgConfig, StateView, }; -use evm_fork_cache::events::erc20::Erc20TransferDecoder; use evm_fork_cache::{PurgeScope, SlotDelta, StateUpdate}; // --------------------------------------------------------------------------- @@ -153,12 +153,12 @@ fn decoder_returns_empty_for_unrecognized_log() { /// Build an ERC-20 `Transfer(from, to, value)` log. fn transfer_log(token: Address, from: Address, to: Address, value: U256) -> Log { let sig = keccak256(b"Transfer(address,address,uint256)"); - let topics = vec![ - sig, - from.into_word(), - to.into_word(), - ]; - Log::new_unchecked(token, topics, Bytes::copy_from_slice(&value.to_be_bytes::<32>())) + let topics = vec![sig, from.into_word(), to.into_word()]; + Log::new_unchecked( + token, + topics, + Bytes::copy_from_slice(&value.to_be_bytes::<32>()), + ) } #[test] @@ -174,7 +174,11 @@ fn erc20_transfer_decodes_to_sub_and_add_deltas() { assert_eq!( updates, vec![ - StateUpdate::slot_delta(token, mapping_slot(from, 3), SlotDelta::Sub(U256::from(100))), + StateUpdate::slot_delta( + token, + mapping_slot(from, 3), + SlotDelta::Sub(U256::from(100)) + ), StateUpdate::slot_delta(token, mapping_slot(to, 3), SlotDelta::Add(U256::from(100))), ] ); @@ -229,13 +233,27 @@ fn erc20_uses_per_token_slot_override_else_default() { &transfer_log(token_default, Address::ZERO, holder, U256::from(1)), &view, ); - assert_eq!(d[0], StateUpdate::slot_delta(token_default, mapping_slot(holder, 3), SlotDelta::Add(U256::from(1)))); + assert_eq!( + d[0], + StateUpdate::slot_delta( + token_default, + mapping_slot(holder, 3), + SlotDelta::Add(U256::from(1)) + ) + ); let c = decoder.decode( &transfer_log(token_custom, Address::ZERO, holder, U256::from(1)), &view, ); - assert_eq!(c[0], StateUpdate::slot_delta(token_custom, mapping_slot(holder, 9), SlotDelta::Add(U256::from(1)))); + assert_eq!( + c[0], + StateUpdate::slot_delta( + token_custom, + mapping_slot(holder, 9), + SlotDelta::Add(U256::from(1)) + ) + ); } #[test] @@ -243,7 +261,10 @@ fn erc20_non_transfer_log_decodes_to_empty() { let decoder = Erc20TransferDecoder::new(U256::from(3)); let view = empty_view(); // Wrong topic0. - let log = bare_log(Address::repeat_byte(0x28), vec![keccak256(b"Approval(address,address,uint256)")]); + let log = bare_log( + Address::repeat_byte(0x28), + vec![keccak256(b"Approval(address,address,uint256)")], + ); assert!(decoder.decode(&log, &view).is_empty()); } @@ -344,7 +365,10 @@ async fn ingest_records_touched_slots() -> Result<()> { let digest = pipeline.ingest_logs(&mut cache, 10, &[bare_log(token, vec![])]); assert!(digest.touched_slots.contains(&(token, U256::from(5)))); - assert_eq!(cache.cached_storage_value(token, U256::from(5)), Some(U256::from(42))); + assert_eq!( + cache.cached_storage_value(token, U256::from(5)), + Some(U256::from(42)) + ); Ok(()) } @@ -379,9 +403,21 @@ async fn reorg_to_purges_addresses_touched_after_head() -> Result<()> { // Reorg back to block 10: purge B and C (touched after 10), keep A. let diff = pipeline.reorg_to(&mut cache, 10); - assert_eq!(backend_slot(&cache, token_a, slot), Some(U256::from(77)), "A untouched"); - assert_eq!(backend_slot(&cache, token_b, slot), None, "B storage purged"); - assert_eq!(backend_slot(&cache, token_c, slot), None, "C storage purged"); + assert_eq!( + backend_slot(&cache, token_a, slot), + Some(U256::from(77)), + "A untouched" + ); + assert_eq!( + backend_slot(&cache, token_b, slot), + None, + "B storage purged" + ); + assert_eq!( + backend_slot(&cache, token_c, slot), + None, + "C storage purged" + ); // The returned diff records the purges (B and C only). let purged: Vec
= diff.purged.iter().map(|r| r.address).collect(); @@ -410,7 +446,11 @@ async fn reorg_config_account_scope_fully_drops_account() -> Result<()> { pipeline.ingest_logs(&mut cache, 20, &[bare_log(token, vec![])]); let diff = pipeline.reorg_to(&mut cache, 19); - assert!(diff.purged.iter().any(|r| r.address == token && r.account_removed)); + assert!( + diff.purged + .iter() + .any(|r| r.address == token && r.account_removed) + ); Ok(()) } @@ -430,10 +470,16 @@ async fn reconcile_reports_mismatch_and_corrects() -> Result<()> { })); let mut pipeline = EventPipeline::new(registry); pipeline.ingest_logs(&mut cache, 1, &[bare_log(token, vec![])]); - assert_eq!(cache.cached_storage_value(token, slot), Some(U256::from(50))); + assert_eq!( + cache.cached_storage_value(token, slot), + Some(U256::from(50)) + ); // Chain truth is 100. Reconcile must surface the drift AND correct the cache. - cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([((token, slot), U256::from(100))]))); + cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([( + (token, slot), + U256::from(100), + )]))); let report = pipeline.reconcile(&mut cache, &[(token, slot)])?; assert_eq!(report.checked, 1); @@ -441,7 +487,10 @@ async fn reconcile_reports_mismatch_and_corrects() -> Result<()> { assert_eq!(report.mismatched[0].old, U256::from(50)); assert_eq!(report.mismatched[0].new, U256::from(100)); // Cache corrected to chain truth. - assert_eq!(cache.cached_storage_value(token, slot), Some(U256::from(100))); + assert_eq!( + cache.cached_storage_value(token, slot), + Some(U256::from(100)) + ); Ok(()) } @@ -461,7 +510,10 @@ async fn reconcile_empty_when_event_state_matches_chain() -> Result<()> { pipeline.ingest_logs(&mut cache, 1, &[bare_log(token, vec![])]); // Chain agrees (100). - cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([((token, slot), U256::from(100))]))); + cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([( + (token, slot), + U256::from(100), + )]))); let report = pipeline.reconcile(&mut cache, &[(token, slot)])?; assert!(report.mismatched.is_empty()); Ok(()) @@ -473,7 +525,11 @@ async fn reconcile_errors_without_fetcher() -> Result<()> { let registry = DecoderRegistry::new(); let mut pipeline = EventPipeline::new(registry); let token = Address::repeat_byte(0x37); - assert!(pipeline.reconcile(&mut cache, &[(token, U256::from(0))]).is_err()); + assert!( + pipeline + .reconcile(&mut cache, &[(token, U256::from(0))]) + .is_err() + ); Ok(()) } @@ -484,11 +540,11 @@ async fn reconcile_errors_without_fetcher() -> Result<()> { #[cfg(feature = "protocols")] mod uniswap_v3 { use super::*; - use alloy_primitives::aliases::{I24, U160}; use alloy_primitives::I256; + use alloy_primitives::aliases::{I24, U160}; use alloy_sol_types::{SolEvent, sol}; use evm_fork_cache::cache::{ - V3_LIQUIDITY_SLOT, V3_SLOT0_SLOT, V3_TICKS_BASE_SLOT, V3_TICK_BITMAP_BASE_SLOT, + V3_LIQUIDITY_SLOT, V3_SLOT0_SLOT, V3_TICK_BITMAP_BASE_SLOT, V3_TICKS_BASE_SLOT, v3_tick_bitmap_storage_key_with_base, v3_tick_info_storage_keys_with_base, }; use evm_fork_cache::events::uniswap_v3::{UniswapV3Decoder, UniswapV3Layout}; @@ -564,7 +620,8 @@ mod uniswap_v3 { fn pool_with_decoder(tick_spacing: i32) -> (Address, UniswapV3Decoder) { let pool = Address::repeat_byte(0x40); - let decoder = UniswapV3Decoder::new().with_pool(pool, UniswapV3Layout::uniswap(tick_spacing)); + let decoder = + UniswapV3Decoder::new().with_pool(pool, UniswapV3Layout::uniswap(tick_spacing)); (pool, decoder) } @@ -578,7 +635,9 @@ mod uniswap_v3 { // observation index (bits 184+). high = unlocked(bit 56 of high) | obs(=7). let high = (U256::from(1) << 56) | U256::from(7); let seeded = pack_slot0(1_000_000, 50, high); - cache.db_mut().insert_account_storage(pool, V3_SLOT0_SLOT, seeded)?; + cache + .db_mut() + .insert_account_storage(pool, V3_SLOT0_SLOT, seeded)?; let mut registry = DecoderRegistry::new(); registry.register(Arc::new(decoder)); @@ -601,9 +660,11 @@ mod uniswap_v3 { let (pool, decoder) = pool_with_decoder(1); let mut cache = setup_cache().await?; install_mock_erc20(&mut cache, pool); - cache - .db_mut() - .insert_account_storage(pool, V3_SLOT0_SLOT, pack_slot0(1, 0, U256::from(1) << 56))?; + cache.db_mut().insert_account_storage( + pool, + V3_SLOT0_SLOT, + pack_slot0(1, 0, U256::from(1) << 56), + )?; let mut registry = DecoderRegistry::new(); registry.register(Arc::new(decoder)); @@ -650,8 +711,10 @@ mod uniswap_v3 { let lo_key = v3_tick_info_storage_keys_with_base(lo, V3_TICKS_BASE_SLOT)[0]; let hi_key = v3_tick_info_storage_keys_with_base(hi, V3_TICKS_BASE_SLOT)[0]; - let (lo_gross, lo_net) = unpack_tick_word(cache.cached_storage_value(pool, lo_key).unwrap()); - let (hi_gross, hi_net) = unpack_tick_word(cache.cached_storage_value(pool, hi_key).unwrap()); + let (lo_gross, lo_net) = + unpack_tick_word(cache.cached_storage_value(pool, lo_key).unwrap()); + let (hi_gross, hi_net) = + unpack_tick_word(cache.cached_storage_value(pool, hi_key).unwrap()); assert_eq!(lo_gross, 500); assert_eq!(lo_net, 500, "lower tick: net += amount"); @@ -677,8 +740,14 @@ mod uniswap_v3 { let lo3 = v3_tick_info_storage_keys_with_base(lo, V3_TICKS_BASE_SLOT)[3]; let hi3 = v3_tick_info_storage_keys_with_base(hi, V3_TICKS_BASE_SLOT)[3]; let init_bit = U256::from(1) << 248; - assert_eq!(cache.cached_storage_value(pool, lo3).unwrap() & init_bit, init_bit); - assert_eq!(cache.cached_storage_value(pool, hi3).unwrap() & init_bit, init_bit); + assert_eq!( + cache.cached_storage_value(pool, lo3).unwrap() & init_bit, + init_bit + ); + assert_eq!( + cache.cached_storage_value(pool, hi3).unwrap() & init_bit, + init_bit + ); // bitmap word 0 (ticks 10 & 20 with tick_spacing 1 → word 0, bits 10 & 20). let word_key = v3_tick_bitmap_storage_key_with_base(0, V3_TICK_BITMAP_BASE_SLOT); @@ -708,17 +777,24 @@ mod uniswap_v3 { ); let lo_key = v3_tick_info_storage_keys_with_base(lo, V3_TICKS_BASE_SLOT)[0]; - let (lo_gross, lo_net) = unpack_tick_word(cache.cached_storage_value(pool, lo_key).unwrap()); + let (lo_gross, lo_net) = + unpack_tick_word(cache.cached_storage_value(pool, lo_key).unwrap()); assert_eq!(lo_gross, 0, "gross back to zero"); assert_eq!(lo_net, 0, "net back to zero"); // initialized flag cleared and bitmap bit cleared. let lo3 = v3_tick_info_storage_keys_with_base(lo, V3_TICKS_BASE_SLOT)[3]; let init_bit = U256::from(1) << 248; - assert_eq!(cache.cached_storage_value(pool, lo3).unwrap_or(U256::ZERO) & init_bit, U256::ZERO); + assert_eq!( + cache.cached_storage_value(pool, lo3).unwrap_or(U256::ZERO) & init_bit, + U256::ZERO + ); let word_key = v3_tick_bitmap_storage_key_with_base(0, V3_TICK_BITMAP_BASE_SLOT); assert_eq!( - cache.cached_storage_value(pool, word_key).unwrap_or(U256::ZERO) & (U256::from(1) << 10), + cache + .cached_storage_value(pool, word_key) + .unwrap_or(U256::ZERO) + & (U256::from(1) << 10), U256::ZERO ); Ok(()) @@ -795,9 +871,16 @@ mod uniswap_v3 { let mut pipeline = EventPipeline::new(registry); let digest = pipeline.ingest_logs(&mut cache, 1, &[mint_log(pool, 10, 20, 500)]); - assert!(digest.applied.has_skipped(), "cold tick words surfaced as skips"); + assert!( + digest.applied.has_skipped(), + "cold tick words surfaced as skips" + ); let lo_key = v3_tick_info_storage_keys_with_base(10, V3_TICKS_BASE_SLOT)[0]; - assert_eq!(cache.cached_storage_value(pool, lo_key), None, "nothing written"); + assert_eq!( + cache.cached_storage_value(pool, lo_key), + None, + "nothing written" + ); Ok(()) } diff --git a/tests/state_update.rs b/tests/state_update.rs index 3647b8e..2ee024e 100644 --- a/tests/state_update.rs +++ b/tests/state_update.rs @@ -22,8 +22,8 @@ use common::{ }; use evm_fork_cache::cache::EvmCache; use evm_fork_cache::{ - AccountPatch, PurgeScope, SkippedBalanceDelta, SkippedDelta, SkippedMask, SlotChange, SlotDelta, - StateDiff, StateUpdate, + AccountPatch, PurgeScope, SkippedBalanceDelta, SkippedDelta, SkippedMask, SlotChange, + SlotDelta, StateDiff, StateUpdate, }; use revm::state::{AccountInfo, Bytecode}; @@ -1666,7 +1666,10 @@ async fn slot_masked_noop_when_masked_bits_already_equal() -> Result<()> { U256::from(0x42), )); - assert!(diff.is_empty(), "masked write that changes nothing is a no-op"); + assert!( + diff.is_empty(), + "masked write that changes nothing is a no-op" + ); assert!(diff.skipped_masks.is_empty()); Ok(()) } @@ -1749,8 +1752,12 @@ async fn slot_masked_full_mask_equals_absolute_on_hot_but_skips_cold() -> Result .db_mut() .insert_account_storage(token, hot, U256::from(7))?; - let hot_diff = - cache.apply_update(&StateUpdate::slot_masked(token, hot, U256::MAX, U256::from(99))); + let hot_diff = cache.apply_update(&StateUpdate::slot_masked( + token, + hot, + U256::MAX, + U256::from(99), + )); assert_eq!(cache.cached_storage_value(token, hot), Some(U256::from(99))); assert_eq!(hot_diff.slots.len(), 1); assert!(hot_diff.skipped_masks.is_empty()); @@ -1769,8 +1776,12 @@ async fn slot_masked_full_mask_equals_absolute_on_hot_but_skips_cold() -> Result #[tokio::test] async fn slot_masked_serde_round_trips() -> Result<()> { - let update = - StateUpdate::slot_masked(Address::repeat_byte(0x17), U256::from(5), U256::from(0xFF), U256::from(3)); + let update = StateUpdate::slot_masked( + Address::repeat_byte(0x17), + U256::from(5), + U256::from(0xFF), + U256::from(3), + ); let json = serde_json::to_string(&update)?; let back: StateUpdate = serde_json::from_str(&json)?; assert_eq!(update, back); From 5f28db2d44193fbf9954e39343a1820a93ce8cfc Mon Sep 17 00:00:00 2001 From: Kai Aldag Date: Tue, 16 Jun 2026 13:22:49 +0100 Subject: [PATCH 17/26] Phase 4 steps 2-5: events module, decoders, pipeline - events/mod.rs (generic core): StateView, EventDecoder, DecoderRegistry, EventPipeline (ingest_logs log-by-log, reorg_to purge-and-resync, reconcile correct+alarm, derived_slots), BlockDigest, ReconcileReport, ReorgConfig, and the async LogSource/drive convenience. - events/erc20.rs (generic core): Erc20TransferDecoder -> Sub/Add balance SlotDeltas, skipping the zero-address leg; reuses parse_transfer. - events/uniswap_v3.rs (protocols): UniswapV3Decoder/UniswapV3Layout. Swap -> masked slot0 (preserves unlocked/observation bits) + absolute liquidity; Mint/Burn -> per-tick gross/net, initialized flag, tickBitmap bit, global liquidity (in-range), all cold-aware via StateView with a mask==MAX,value==0 could-not-compute marker. - cache/mod.rs: impl StateView for EvmCache; refactor verify_slots into verify_slots_inner (+fetched_ok count) and add reconcile_slots, which errors on a total fetch failure (honest-freshness) so the pipeline's reconcile surfaces an unverifiable re-read rather than a false all-clear. - lib.rs: pub mod events + re-exports. - tests/event_pipeline.rs: #[allow(dead_code)] on the unused tick_word test helper (no assertion/behaviour change) so clippy --all-targets -D warnings stays clean. All 24 event_pipeline tests + existing suites green on both feature configs. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/cache/mod.rs | 49 ++++- src/events/erc20.rs | 145 +++++++++++++ src/events/mod.rs | 439 +++++++++++++++++++++++++++++++++++++++ src/events/uniswap_v3.rs | 399 +++++++++++++++++++++++++++++++++++ src/lib.rs | 19 +- tests/event_pipeline.rs | 1 + 6 files changed, 1048 insertions(+), 4 deletions(-) create mode 100644 src/events/erc20.rs create mode 100644 src/events/mod.rs create mode 100644 src/events/uniswap_v3.rs diff --git a/src/cache/mod.rs b/src/cache/mod.rs index b20f387..7387e82 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -1804,8 +1804,20 @@ impl EvmCache { /// none is available. This is the synchronous main-thread primitive; the /// background validator performs the equivalent comparison against a snapshot. pub fn verify_slots(&mut self, slots: &[(Address, U256)]) -> Result> { + Ok(self.verify_slots_inner(slots)?.0) + } + + /// Shared implementation for [`verify_slots`](Self::verify_slots) and the + /// pipeline's reconcile path. Returns `(changed, fetched_ok)` where + /// `fetched_ok` is the number of requested slots the fetcher returned a value + /// for (failed per-slot fetches are skipped, not errors). Errors only when no + /// batch fetcher is configured. + fn verify_slots_inner( + &mut self, + slots: &[(Address, U256)], + ) -> Result<(Vec, usize)> { if slots.is_empty() { - return Ok(Vec::new()); + return Ok((Vec::new(), 0)); } let fetcher = self .storage_batch_fetcher @@ -1824,6 +1836,7 @@ impl EvmCache { let mut changed = Vec::new(); let mut to_inject = Vec::new(); + let mut fetched_ok = 0usize; for (addr, slot, fetched) in results { let fresh = match fetched { Ok(value) => value, @@ -1832,6 +1845,7 @@ impl EvmCache { continue; } }; + fetched_ok += 1; // A slot the cache never saw is treated as old = ZERO (the value a // sim would have read), so a non-zero fresh value counts as a change. let old = cached @@ -1853,6 +1867,29 @@ impl EvmCache { if !to_inject.is_empty() { self.inject_storage_batch_fresh(&to_inject); } + Ok((changed, fetched_ok)) + } + + /// Reconciliation re-read used by [`EventPipeline::reconcile`](crate::events::EventPipeline::reconcile). + /// + /// Like [`verify_slots`](Self::verify_slots) it fetches the requested slots, + /// injects the ones that changed, and returns the changed set — but it is + /// **honest about reachability**: it errors not only when no batch fetcher is + /// configured, but also when a non-empty request could not fetch **any** slot + /// (a total fetch failure — e.g. the default RPC fetcher invoked with no usable + /// runtime, or an unreachable provider). Reconciliation that silently "verified + /// nothing" would be a false all-clear, so it surfaces as an error for the + /// caller to retry. A partially-successful fetch returns `Ok` with whatever + /// changed. + pub fn reconcile_slots(&mut self, slots: &[(Address, U256)]) -> Result> { + let (changed, fetched_ok) = self.verify_slots_inner(slots)?; + if !slots.is_empty() && fetched_ok == 0 { + return Err(anyhow!( + "reconcile could not fetch any of the {} requested slot(s) \ + (no usable storage fetcher / provider unreachable)", + slots.len() + )); + } Ok(changed) } @@ -3499,6 +3536,16 @@ impl EvmCache { } } +/// Read-only state view for the event pipeline (Pillar B.2): a decoder reads the +/// current cached value of a slot through [`cached_storage_value`](EvmCache::cached_storage_value), +/// which never touches RPC and is `account_state`-aware (a cold slot reads +/// `None`). +impl crate::events::StateView for EvmCache { + fn storage(&self, address: Address, slot: U256) -> Option { + self.cached_storage_value(address, slot) + } +} + impl EvmCache { /// Create a LocalContext that reuses the shared memory buffer. /// diff --git a/src/events/erc20.rs b/src/events/erc20.rs new file mode 100644 index 0000000..c9c467f --- /dev/null +++ b/src/events/erc20.rs @@ -0,0 +1,145 @@ +//! Generic ERC-20 `Transfer` decoder (generic core). +//! +//! [`Erc20TransferDecoder`] turns a standard ERC-20 +//! `Transfer(from, to, value)` log into two relative balance updates — a +//! [`SlotDelta::Sub`] on the sender's balance slot and a +//! [`SlotDelta::Add`] on the recipient's — so the cache +//! tracks balances from the event stream without ever reading the resulting +//! absolute balances. It is the log-driven form of the Phase 3 reactive-balance +//! case. +//! +//! # Balance-slot derivation +//! +//! An ERC-20 `balanceOf` is a `mapping(address => uint256)` at some base slot. +//! The decoder hashes the owner into that mapping the canonical Solidity way: +//! `keccak256(abi.encode(owner, balance_slot))`. The base slot is configurable +//! per token ([`with_token`](Erc20TransferDecoder::with_token)) with a default +//! fallback ([`new`](Erc20TransferDecoder::new)), since different tokens place +//! `balanceOf` at different slots. +//! +//! # Mint / burn legs +//! +//! A mint (`from == 0`) or burn (`to == 0`) has no real holder on the +//! zero-address leg, so that leg is **skipped** — only the non-zero side emits a +//! delta. Cold balances follow the Phase 3 contract: the +//! [`SlotDelta`] is skipped at apply time and surfaced in +//! [`StateDiff::skipped`](crate::StateDiff::skipped) (the caller seeds the +//! balance, or the next read lazily fetches it). The decoder ignores the +//! [`StateView`] — it is stateless. + +use std::collections::HashMap; + +use alloy_primitives::{Address, Log, U256, keccak256}; +use alloy_sol_types::SolValue; + +use crate::events::{EventDecoder, StateView}; +use crate::inspector::TransferInspector; +use crate::state_update::{SlotDelta, StateUpdate}; + +/// Decodes ERC-20 `Transfer` logs into relative balance [`SlotDelta`] updates. +/// +/// ``` +/// use alloy_primitives::{Address, Bytes, Log, U256, keccak256}; +/// use alloy_sol_types::SolValue; +/// use evm_fork_cache::events::{EventDecoder, StateView}; +/// use evm_fork_cache::events::erc20::Erc20TransferDecoder; +/// use evm_fork_cache::{SlotDelta, StateUpdate}; +/// +/// // A read-only view that reports every slot cold (decoder is stateless anyway). +/// struct ColdView; +/// impl StateView for ColdView { +/// fn storage(&self, _: Address, _: U256) -> Option { None } +/// } +/// +/// let token = Address::repeat_byte(0x20); +/// let from = Address::repeat_byte(0x21); +/// let to = Address::repeat_byte(0x22); +/// +/// // Transfer(from, to, 100) log: balanceOf mapping at slot 3. +/// let sig = keccak256(b"Transfer(address,address,uint256)"); +/// let log = Log::new_unchecked( +/// token, +/// vec![sig, from.into_word(), to.into_word()], +/// Bytes::copy_from_slice(&U256::from(100).to_be_bytes::<32>()), +/// ); +/// +/// let decoder = Erc20TransferDecoder::new(U256::from(3)); +/// let updates = decoder.decode(&log, &ColdView); +/// +/// let slot = |owner: Address| { +/// U256::from_be_bytes(keccak256((owner, U256::from(3)).abi_encode()).0) +/// }; +/// assert_eq!(updates, vec![ +/// StateUpdate::slot_delta(token, slot(from), SlotDelta::Sub(U256::from(100))), +/// StateUpdate::slot_delta(token, slot(to), SlotDelta::Add(U256::from(100))), +/// ]); +/// ``` +pub struct Erc20TransferDecoder { + /// Balance mapping base slot per token (the `balanceOf` mapping's slot). + balance_slots: HashMap, + /// Fallback balance mapping base slot for tokens not in the map. + default_balance_slot: U256, +} + +impl Erc20TransferDecoder { + /// Create a decoder with `default_balance_slot` as the `balanceOf` mapping + /// base slot for any token without a per-token override. + pub fn new(default_balance_slot: U256) -> Self { + Self { + balance_slots: HashMap::new(), + default_balance_slot, + } + } + + /// Override the `balanceOf` mapping base slot for `token` (builder style). + pub fn with_token(mut self, token: Address, balance_slot: U256) -> Self { + self.balance_slots.insert(token, balance_slot); + self + } + + /// The configured balance mapping base slot for `token` (its override, else + /// the default). + fn balance_slot(&self, token: Address) -> U256 { + self.balance_slots + .get(&token) + .copied() + .unwrap_or(self.default_balance_slot) + } +} + +/// The hashed storage slot of `balanceOf[owner]` for a `mapping(address => +/// uint256)` at `mapping_slot`. +fn balance_key(owner: Address, mapping_slot: U256) -> U256 { + U256::from_be_bytes(keccak256((owner, mapping_slot).abi_encode()).0) +} + +impl EventDecoder for Erc20TransferDecoder { + fn decode(&self, log: &Log, _view: &dyn StateView) -> Vec { + // Reuse the canonical ERC-20 Transfer signature match + topic/data decode. + // Returns None for a non-Transfer log (wrong topic0, <3 topics, <32 data + // bytes). + let Some(transfer) = TransferInspector::parse_transfer(log) else { + return Vec::new(); + }; + + let slot = self.balance_slot(transfer.token); + let mut updates = Vec::with_capacity(2); + + // Skip the zero-address leg (mint = from == 0, burn = to == 0). + if transfer.from != Address::ZERO { + updates.push(StateUpdate::slot_delta( + transfer.token, + balance_key(transfer.from, slot), + SlotDelta::Sub(transfer.value), + )); + } + if transfer.to != Address::ZERO { + updates.push(StateUpdate::slot_delta( + transfer.token, + balance_key(transfer.to, slot), + SlotDelta::Add(transfer.value), + )); + } + updates + } +} diff --git a/src/events/mod.rs b/src/events/mod.rs new file mode 100644 index 0000000..510c558 --- /dev/null +++ b/src/events/mod.rs @@ -0,0 +1,439 @@ +//! Event → state pipeline (Pillar B.2 — the *reader half* of the event pipeline). +//! +//! Phase 3 ([`state_update`](crate::state_update)) built the *writer half*: the +//! generic [`StateUpdate`] vocabulary and the cold-aware +//! [`apply_updates`](crate::cache::EvmCache::apply_updates) that consumes it. This +//! module builds the *reader half*: it turns an on-chain [`Log`] into that same +//! vocabulary and drives it through the cache, keeping event-derived state +//! reactively fresh. +//! +//! # The flow +//! +//! ```text +//! Log ─▶ EventDecoder::decode(log, &StateView) ─▶ Vec +//! │ +//! apply_updates ▼ +//! EvmCache (+ StateDiff) +//! ``` +//! +//! A [`DecoderRegistry`] dispatches a log to the decoders registered for its +//! emitting address (plus any global decoders) and concatenates their output. An +//! [`EventPipeline`] orchestrates a block's logs: [`ingest_logs`] decodes and +//! applies them **log-by-log in order** (so a later log's decode observes the +//! effects of earlier ones through the [`StateView`]), [`reorg_to`] purges the +//! addresses touched after a new head, and [`reconcile`] re-reads sampled +//! event-derived slots against chain truth (correct **and** alarm). +//! +//! [`ingest_logs`]: EventPipeline::ingest_logs +//! [`reorg_to`]: EventPipeline::reorg_to +//! [`reconcile`]: EventPipeline::reconcile +//! +//! # Decoders are pure data functions +//! +//! [`EventDecoder::decode`] is a pure function of `(log, pre-state)`: it performs +//! no I/O and emits serializable, replayable [`StateUpdate`] data. Most updates +//! need no pre-state ([`SlotDelta`](crate::StateUpdate::SlotDelta) and +//! [`SlotMasked`](crate::StateUpdate::SlotMasked) are read-modify-write *at apply +//! time*), but stateful adapters — UniswapV3 tick maintenance must read the +//! current `liquidityGross`/`liquidityNet`/`tick`/bitmap to recompute a packed +//! word — read the narrow read-only [`StateView`]. The view never touches RPC; a +//! slot absent from the cache reads `None` (cold), and a decoder that cannot +//! compute against a cold word surfaces a skip rather than inventing a value. +//! +//! # `!Send` cache discipline +//! +//! [`EvmCache`] is `!Send` (it owns the mutable fork and +//! blocks on RPC internally). All of [`EventPipeline`]'s core methods +//! ([`ingest_logs`](EventPipeline::ingest_logs) / +//! [`reorg_to`](EventPipeline::reorg_to) / +//! [`reconcile`](EventPipeline::reconcile)) take `&mut EvmCache` and are +//! **synchronous** — they never `.await`, so the cache is never held across a +//! yield point. This is what makes the core deterministically testable offline. +//! The async [`drive`] convenience holds the cache only across the *log source* +//! await (the source future is `Send`; the cache is untouched during it). +//! +//! # Freshness wiring +//! +//! [`BlockDigest::touched_slots`] surfaces the `(address, slot)` set written for a +//! block so a caller can classify event-derived slots in a +//! [`FreshnessRegistry`](crate::freshness::FreshnessRegistry) — typically pin them +//! ([`Validity::Pinned`](crate::freshness::Validity::Pinned)) or mark them +//! [`Validity::ValidThrough`](crate::freshness::Validity::ValidThrough) so the +//! optimistic validator does not waste RPC re-verifying state the pipeline keeps +//! fresh — then call +//! [`FreshnessController::on_new_block`](crate::freshness::FreshnessController::on_new_block). +//! No controller internals change. Periodically call +//! [`reconcile`](EventPipeline::reconcile) to sample-check those slots against the +//! chain (honest freshness). + +pub mod erc20; +#[cfg(feature = "protocols")] +#[cfg_attr(docsrs, doc(cfg(feature = "protocols")))] +pub mod uniswap_v3; + +use std::collections::{HashMap, HashSet, VecDeque}; +use std::sync::Arc; + +use alloy_primitives::{Address, Log, U256}; +use anyhow::Result; + +use crate::cache::EvmCache; +use crate::freshness::SlotChange; +use crate::state_update::{PurgeScope, StateDiff, StateUpdate}; + +/// Read-only view of current cached state handed to a decoder. +/// +/// Decoders that compute post-state from pre-state (e.g. UniswapV3 tick +/// maintenance) read through this; stateless decoders (ERC-20 `Transfer`, V3 +/// `Swap`) ignore it. The view never touches RPC — a slot absent from the cache +/// reads `None`. +pub trait StateView { + /// Current cached value of `(address, slot)` (overlay ▸ backend ▸ `None`), + /// matching what the EVM would `SLOAD` (`account_state`-aware). `None` means + /// the slot is **cold** — neither cache layer has seen it. + fn storage(&self, address: Address, slot: U256) -> Option; +} + +/// Decode one log into zero or more targeted [`StateUpdate`]s. +/// +/// `decode` is a pure function of `(log, pre-state)`: it performs no I/O and emits +/// data (the updates are serializable and replayable against matching pre-state). +/// The pipeline applies the result through +/// [`apply_updates`](crate::cache::EvmCache::apply_updates). +/// +/// A decoder returns `vec![]` for any log it does not recognise (wrong topic0, an +/// unregistered emitting address, a malformed payload). The pipeline counts a log +/// as *decoded* only when some decoder produced at least one update for it. +pub trait EventDecoder: Send + Sync { + /// Decode `log` against the read-only pre-state `view` into targeted updates. + fn decode(&self, log: &Log, view: &dyn StateView) -> Vec; +} + +/// Dispatches a log to the decoders registered for its emitting address (and any +/// global decoders) and concatenates their output. +/// +/// Dispatch is by emitting address ([`Log::address`]); topic0 filtering is each +/// decoder's own concern (a decoder returns `vec![]` for a log it does not +/// recognise). Address-scoped decoders are consulted first, then global ones, and +/// the per-decoder outputs are concatenated in that order. +#[derive(Default)] +pub struct DecoderRegistry { + /// Decoders consulted for every log, in registration order. + global: Vec>, + /// Decoders consulted only for logs emitted by a specific address. + per_address: HashMap>>, +} + +impl DecoderRegistry { + /// Create an empty registry with no decoders. + pub fn new() -> Self { + Self::default() + } + + /// Register a decoder consulted for **every** log. + pub fn register(&mut self, decoder: Arc) -> &mut Self { + self.global.push(decoder); + self + } + + /// Register a decoder consulted only for logs emitted by `address`. + pub fn register_for_address( + &mut self, + address: Address, + decoder: Arc, + ) -> &mut Self { + self.per_address.entry(address).or_default().push(decoder); + self + } + + /// Decode `log` through every applicable decoder, concatenating the results + /// (address-scoped decoders first, then global), preserving order. + pub fn decode(&self, log: &Log, view: &dyn StateView) -> Vec { + let mut out = Vec::new(); + if let Some(scoped) = self.per_address.get(&log.address) { + for decoder in scoped { + out.extend(decoder.decode(log, view)); + } + } + for decoder in &self.global { + out.extend(decoder.decode(log, view)); + } + out + } +} + +/// How a reorg purges the addresses touched after the new head. +/// +/// `depth` bounds the per-block touched-address history retained for reorg purge +/// (the reorg horizon); older entries are dropped as new blocks are ingested. +/// `scope` is the [`PurgeScope`] applied to each touched address on +/// [`reorg_to`](EventPipeline::reorg_to). +#[derive(Clone, Debug)] +pub struct ReorgConfig { + /// How many recent blocks of touched-address history to retain for reorg + /// purge (the reorg horizon). Older entries are dropped. + pub depth: usize, + /// Purge scope used on reorg. The default ([`PurgeScope::AllStorage`]) drops + /// storage so it re-fetches but keeps the account header; + /// [`PurgeScope::Account`] drops the whole account. + pub scope: PurgeScope, +} + +impl Default for ReorgConfig { + fn default() -> Self { + Self { + depth: 64, + scope: PurgeScope::AllStorage, + } + } +} + +/// Per-block result of [`EventPipeline::ingest_logs`]. +#[derive(Clone, Debug, Default)] +pub struct BlockDigest { + /// The block whose logs were ingested. + pub block: u64, + /// Merged diff of everything applied for the block (changes-only **and** + /// skips — check [`StateDiff::has_skipped`]). + pub applied: StateDiff, + /// Number of logs that decoded to at least one update. + pub decoded_logs: usize, + /// The `(address, slot)` set written this block (for freshness + /// classification — see the module docs). + pub touched_slots: Vec<(Address, U256)>, +} + +/// Result of [`EventPipeline::reconcile`]. +#[derive(Clone, Debug, Default)] +pub struct ReconcileReport { + /// How many slots were sampled. + pub checked: usize, + /// Slots whose event-derived value disagreed with chain truth. A non-empty + /// list is a **drift alarm**: the cache had drifted and + /// [`verify_slots`](crate::cache::EvmCache::verify_slots) has now injected the + /// fresh chain values (correct + alarm). + pub mismatched: Vec, +} + +/// Orchestrates decoding, applying, reorg handling, and reconciliation of a +/// block's logs against an [`EvmCache`]. +/// +/// Construct one from a [`DecoderRegistry`], then call +/// [`ingest_logs`](Self::ingest_logs) per block. See the [module docs](crate::events) +/// for the freshness-wiring pattern (event-derived slots → +/// [`Pinned`](crate::freshness::Validity::Pinned), reconciled periodically). +pub struct EventPipeline { + registry: DecoderRegistry, + reorg: ReorgConfig, + /// Ring of `(block, touched addresses)` for reorg purge, newest at the back, + /// bounded to `reorg.depth`. + touched: VecDeque<(u64, Vec
)>, + /// Every event-derived `(address, slot)` seen so far (reconcile sampling + /// source). + derived_slots: HashSet<(Address, U256)>, +} + +impl EventPipeline { + /// Create a pipeline over `registry` with the default [`ReorgConfig`]. + pub fn new(registry: DecoderRegistry) -> Self { + Self { + registry, + reorg: ReorgConfig::default(), + touched: VecDeque::new(), + derived_slots: HashSet::new(), + } + } + + /// Override the [`ReorgConfig`] (reorg horizon depth + purge scope). + pub fn with_reorg_config(mut self, cfg: ReorgConfig) -> Self { + self.reorg = cfg; + self + } + + /// Decode + apply a block's logs, **log-by-log in order**, recording touched + /// state for reorg tracking. Returns the per-block [`BlockDigest`]. + /// + /// Each log is decoded against the *current* cache state and applied + /// immediately, so a later log's decode observes the effects of earlier logs + /// in the same block through the [`StateView`] (e.g. a same-block `Burn` after + /// a `Mint`, or two overlapping `Mint`s). The touched addresses are recorded + /// in the depth-bounded reorg ring under `block`, and the touched + /// `(address, slot)` pairs into the reconcile-sampling set. + pub fn ingest_logs(&mut self, cache: &mut EvmCache, block: u64, logs: &[Log]) -> BlockDigest { + let mut digest = BlockDigest { + block, + ..Default::default() + }; + let mut touched_addrs: HashSet
= HashSet::new(); + + for log in logs { + // Decode against the current cache view (immutable borrow), then drop + // that borrow before taking the &mut borrow for apply. Decode returns + // owned data, so the two borrows never overlap. + let updates = self.registry.decode(log, &*cache); + if updates.is_empty() { + continue; + } + let diff = cache.apply_updates(&updates); + + // A log counts as decoded when it produced at least one update. + digest.decoded_logs += 1; + + // Record touched addresses (for reorg) and touched slots (for + // freshness + reconcile) from every category of the diff. + for change in &diff.slots { + touched_addrs.insert(change.address); + self.note_touched_slot(&mut digest, change.address, change.slot); + } + for change in &diff.accounts { + touched_addrs.insert(change.address); + } + for record in &diff.purged { + touched_addrs.insert(record.address); + } + for skip in &diff.skipped { + touched_addrs.insert(skip.address); + self.note_touched_slot(&mut digest, skip.address, skip.slot); + } + for skip in &diff.skipped_balances { + touched_addrs.insert(skip.address); + } + for skip in &diff.skipped_masks { + touched_addrs.insert(skip.address); + self.note_touched_slot(&mut digest, skip.address, skip.slot); + } + + digest.applied.merge(diff); + } + + if !touched_addrs.is_empty() { + self.touched + .push_back((block, touched_addrs.into_iter().collect())); + self.trim_ring(); + } + + digest + } + + /// Reorg to `new_head`: purge (per [`ReorgConfig::scope`]) every address + /// touched in a block **>** `new_head`, drop those ring entries, and return the + /// merged purge [`StateDiff`]. + /// + /// The next read of a purged address re-fetches from RPC. The caller then + /// re-ingests the canonical chain's logs for the reorged range (and/or the + /// next read lazily re-fetches). + pub fn reorg_to(&mut self, cache: &mut EvmCache, new_head: u64) -> StateDiff { + // Collect the addresses touched strictly after the new head, deduped. + let mut to_purge: HashSet
= HashSet::new(); + for (block, addrs) in &self.touched { + if *block > new_head { + to_purge.extend(addrs.iter().copied()); + } + } + + // Drop the rolled-back ring entries and the derived slots they own. + self.touched.retain(|(block, _)| *block <= new_head); + self.derived_slots + .retain(|(addr, _)| !to_purge.contains(addr)); + + let updates: Vec = to_purge + .into_iter() + .map(|addr| StateUpdate::purge(addr, self.reorg.scope.clone())) + .collect(); + cache.apply_updates(&updates) + } + + /// Sampled reconciliation: re-read `slots` via + /// [`EvmCache::verify_slots`](crate::cache::EvmCache::verify_slots) (correct + + /// alarm). Returns the mismatches. + /// + /// It fetches the fresh chain value for each slot, injects the ones that + /// changed (so the cache is **corrected**), and returns the changed set — a + /// non-empty [`ReconcileReport::mismatched`] is the **drift alarm**: + /// event-derived state had drifted and has now been corrected to chain truth. + /// Honest about reachability (via + /// [`EvmCache::reconcile_slots`](crate::cache::EvmCache::reconcile_slots)): it + /// errors when no batch fetcher is configured **or** when a non-empty request + /// could not fetch any slot (a total fetch failure is not a silent all-clear). + /// An empty `slots` is a no-op that returns an empty report. + pub fn reconcile( + &mut self, + cache: &mut EvmCache, + slots: &[(Address, U256)], + ) -> Result { + let mismatched = cache.reconcile_slots(slots)?; + Ok(ReconcileReport { + checked: slots.len(), + mismatched, + }) + } + + /// All event-derived slots seen so far (the sampling source for + /// [`reconcile`](Self::reconcile)). + pub fn derived_slots(&self) -> impl Iterator + '_ { + self.derived_slots.iter().copied() + } + + /// Record a touched slot in both the per-block digest (deduped within the + /// block) and the global all-time reconcile-sampling set. + fn note_touched_slot(&mut self, digest: &mut BlockDigest, address: Address, slot: U256) { + self.derived_slots.insert((address, slot)); + if !digest.touched_slots.contains(&(address, slot)) { + digest.touched_slots.push((address, slot)); + } + } + + /// Trim the reorg ring to the configured depth, dropping the oldest entries. + fn trim_ring(&mut self) { + while self.touched.len() > self.reorg.depth { + self.touched.pop_front(); + } + } +} + +/// A signalled reorg accompanying a block from a [`LogSource`]. +/// +/// `None` means the block extends the current head; `Some(new_head)` asks the +/// driver to [`reorg_to`](EventPipeline::reorg_to) `new_head` before ingesting. +pub type ReorgSignal = Option; + +/// An async source of blocks of logs for [`drive`]. +/// +/// This is the thin async convenience layer (§7.5): a production WS / +/// `subscribe_logs` adapter implements it; the offline example feeds a vec-backed +/// source. The synchronous [`EventPipeline`] core is the tested contract. +pub trait LogSource { + /// Yield the next block: its number, its logs, and an optional reorg signal. + /// `None` ends the stream. + fn next_block( + &mut self, + ) -> impl std::future::Future, ReorgSignal)>> + Send; +} + +/// Drive `pipeline` over `source`, ingesting each block (reorging first when +/// signalled) and invoking `on_block` after each ingest. +/// +/// A thin async convenience over the synchronous core: it pulls a block from the +/// `Send` source (the only `.await`), then synchronously +/// [`reorg_to`](EventPipeline::reorg_to) (if signalled) and +/// [`ingest_logs`](EventPipeline::ingest_logs), holding the `!Send` cache only +/// across the synchronous section. `on_block` is where a caller wires +/// [`FreshnessController::on_new_block`](crate::freshness::FreshnessController::on_new_block) +/// and freshness classification of the digest's touched slots. +pub async fn drive( + pipeline: &mut EventPipeline, + cache: &mut EvmCache, + mut source: S, + mut on_block: F, +) where + S: LogSource, + F: FnMut(&BlockDigest), +{ + while let Some((block, logs, reorg)) = source.next_block().await { + if let Some(new_head) = reorg { + pipeline.reorg_to(cache, new_head); + } + let digest = pipeline.ingest_logs(cache, block, &logs); + on_block(&digest); + } +} diff --git a/src/events/uniswap_v3.rs b/src/events/uniswap_v3.rs new file mode 100644 index 0000000..c93d667 --- /dev/null +++ b/src/events/uniswap_v3.rs @@ -0,0 +1,399 @@ +//! UniswapV3 / PancakeSwap V3 event adapter (`protocols` feature). +//! +//! [`UniswapV3Decoder`] turns a pool's `Swap` / `Mint` / `Burn` logs into the +//! Phase 3 [`StateUpdate`] vocabulary, maintaining the slots a +//! swap simulation reads: +//! +//! - **`Swap`** (stateless) → a masked `slot0` write (new `sqrtPriceX96` + `tick`, +//! **preserving** the observation index and the `unlocked` flag) plus an +//! absolute `liquidity` write (the event carries post-swap liquidity). +//! - **`Mint`/`Burn`** (stateful, reads the [`StateView`]) → per-tick +//! `liquidityGross` / `liquidityNet`, the `initialized` flag, the `tickBitmap` +//! word bit, and the global `liquidity` (conditional on the current tick). +//! +//! The decoder dispatches by emitting address: a log from a pool not registered +//! via [`with_pool`](UniswapV3Decoder::with_pool) decodes to nothing. It matches +//! events by topic0 (`Swap`/`Mint`/`Burn` signature hashes) and decodes with +//! [`SolEvent`]. +//! +//! # `slot0` bit layout (Uniswap / Pancake) +//! +//! `sqrtPriceX96` = bits [0,160), `tick` (int24) = bits [160,184), and the +//! observation index / cardinality / fee-protocol / **`unlocked`** flag occupy +//! bits [184,256). The `Swap` handler masks the low 184 bits, so the high bits — +//! crucially `unlocked` — survive. Clobbering `unlocked` to 0 would make a +//! subsequent quote/swap revert `LOK`; that is the headline reason `Swap` uses a +//! [`SlotMasked`](crate::StateUpdate::SlotMasked) rather than an absolute write. +//! +//! # Tick word packing +//! +//! Tick slot **+0** packs `liquidityGross` (uint128) = bits [0,128) and +//! `liquidityNet` (int128, two's-complement) = bits [128,256). `Mint` adds +//! `amount` to gross at both ticks and to net at the lower / from net at the +//! upper; `Burn` is the inverse. These are recomputed against the **current** +//! cached word read through the [`StateView`] and emitted as absolute `Slot` +//! writes. The `initialized` flag lives at tick slot **+3**, bit 248 (matching +//! `inject_v3_ticks`); the `tickBitmap` is keyed by the compressed tick +//! `tick / tick_spacing`. +//! +//! # Cold-aware +//! +//! When a needed word is cold ([`StateView::storage`] → `None`), the update is +//! **not** computed against an assumed value — it is skipped and surfaced. Masked +//! sub-word updates (bitmap / initialized) surface as their natural +//! [`SkippedMask`](crate::SkippedMask); the absolute tick-word / global-liquidity +//! writes that cannot be computed surface as a `SkippedMask` with +//! `mask == U256::MAX, value == U256::ZERO` (the "could-not-compute" cold marker — +//! see [`SkippedMask`](crate::SkippedMask)). A pool installed with +//! `StorageCleared` storage reads an unseeded slot as `Some(ZERO)` (hot zero), so +//! tick maintenance proceeds from zero; only a pool with no local account reads +//! cold. +//! +//! # Known limitation (§6.4) +//! +//! Event-derived tick maintenance does **not** reconstruct `feeGrowthOutside0/1X128` +//! (tick slots +1/+2), `secondsOutside`, or oracle observations — these are not +//! derivable from `Mint`/`Burn`/`Swap`. **Swap price/liquidity quoting is +//! unaffected** (the swap-amount math does not depend on `feeGrowthOutside`); fee +//! accounting and `collect` are not maintained. Sampled +//! [`reconcile`](crate::events::EventPipeline::reconcile) and reorg +//! [`reorg_to`](crate::events::EventPipeline::reorg_to) are the backstop. See +//! `KNOWN_ISSUES.md`. + +use std::collections::HashMap; + +use alloy_primitives::{Address, Log, U256}; +use alloy_sol_types::{SolEvent, sol}; + +use crate::cache::{ + PANCAKE_V3_LIQUIDITY_SLOT, PANCAKE_V3_TICK_BITMAP_BASE_SLOT, PANCAKE_V3_TICKS_BASE_SLOT, + V3_LIQUIDITY_SLOT, V3_SLOT0_SLOT, V3_TICK_BITMAP_BASE_SLOT, V3_TICKS_BASE_SLOT, + v3_tick_bitmap_storage_key_with_base, v3_tick_info_storage_keys_with_base, +}; +use crate::events::{EventDecoder, StateView}; +use crate::state_update::StateUpdate; + +sol! { + event Swap(address indexed sender, address indexed recipient, int256 amount0, int256 amount1, uint160 sqrtPriceX96, uint128 liquidity, int24 tick); + event Mint(address sender, address indexed owner, int24 indexed tickLower, int24 indexed tickUpper, uint128 amount, uint256 amount0, uint256 amount1); + event Burn(address indexed owner, int24 indexed tickLower, int24 indexed tickUpper, uint128 amount, uint256 amount0, uint256 amount1); +} + +/// Bit position where the `tick` field starts in a packed `slot0` word. +const SLOT0_TICK_SHIFT: usize = 160; +/// Number of low bits of `slot0` owned by `sqrtPriceX96` ‖ `tick` +/// (`[0,160)` + `[160,184)`); the bits above are preserved by the swap mask. +const SLOT0_PRICE_TICK_BITS: usize = 184; +/// Bit position of the `initialized` flag in tick slot +3. +const TICK_INITIALIZED_BIT: usize = 248; + +/// Per-pool V3 storage layout (slot bases + tick spacing). +/// +/// Uniswap V3 and PancakeSwap V3 share the `slot0` bit layout (only the base slot +/// numbers differ — Pancake's `uint32 feeProtocol` shifts subsequent slots by +1). +/// `tick_spacing` is required for `tickBitmap` word/bit math (the bitmap is keyed +/// by the compressed tick `tick / tick_spacing`). +#[derive(Clone, Debug)] +pub struct UniswapV3Layout { + /// Storage slot of `slot0` (packed price / tick / observation / unlocked). + pub slot0_slot: U256, + /// Storage slot of the global `liquidity`. + pub liquidity_slot: U256, + /// Base slot of the `ticks` mapping (`mapping(int24 => Tick.Info)`). + pub ticks_base_slot: U256, + /// Base slot of the `tickBitmap` mapping (`mapping(int16 => uint256)`). + pub tick_bitmap_base_slot: U256, + /// The pool's `tickSpacing` (for compressed-tick bitmap word/bit math). + pub tick_spacing: i32, +} + +impl UniswapV3Layout { + /// The canonical Uniswap V3 layout for a pool with the given `tick_spacing`. + pub fn uniswap(tick_spacing: i32) -> Self { + Self { + slot0_slot: V3_SLOT0_SLOT, + liquidity_slot: V3_LIQUIDITY_SLOT, + ticks_base_slot: V3_TICKS_BASE_SLOT, + tick_bitmap_base_slot: V3_TICK_BITMAP_BASE_SLOT, + tick_spacing, + } + } + + /// The PancakeSwap V3 layout for a pool with the given `tick_spacing` (slots + /// shifted +1 relative to Uniswap; `slot0` stays at slot 0). + pub fn pancake(tick_spacing: i32) -> Self { + Self { + slot0_slot: V3_SLOT0_SLOT, + liquidity_slot: PANCAKE_V3_LIQUIDITY_SLOT, + ticks_base_slot: PANCAKE_V3_TICKS_BASE_SLOT, + tick_bitmap_base_slot: PANCAKE_V3_TICK_BITMAP_BASE_SLOT, + tick_spacing, + } + } +} + +/// Decodes UniswapV3 / PancakeSwap V3 `Swap` / `Mint` / `Burn` logs into targeted +/// [`StateUpdate`]s. +/// +/// Register pools with [`with_pool`](Self::with_pool); a log from an unregistered +/// pool decodes to nothing. +#[derive(Default)] +pub struct UniswapV3Decoder { + /// Per-pool layout. A log from a pool not in this map decodes to nothing. + pools: HashMap, +} + +impl UniswapV3Decoder { + /// Create an empty decoder with no pools registered. + pub fn new() -> Self { + Self::default() + } + + /// Register `pool` with its storage `layout` (builder style). + pub fn with_pool(mut self, pool: Address, layout: UniswapV3Layout) -> Self { + self.pools.insert(pool, layout); + self + } +} + +/// The cold-tick "could-not-compute" marker: a [`StateUpdate::SlotMasked`] with +/// `mask == U256::MAX, value == U256::ZERO`, which `apply_updates` skip-surfaces +/// for a cold slot (see [`SkippedMask`](crate::SkippedMask)). +fn cold_marker(pool: Address, slot: U256) -> StateUpdate { + StateUpdate::slot_masked(pool, slot, U256::MAX, U256::ZERO) +} + +/// Unpack a tick slot +0 word: `(liquidityGross, liquidityNet)`. +fn unpack_tick_word(word: U256) -> (u128, i128) { + let gross = u128::try_from(word & U256::from(u128::MAX)).unwrap_or(0); + let net = u128::try_from((word >> 128) & U256::from(u128::MAX)).unwrap_or(0) as i128; + (gross, net) +} + +/// Pack `(liquidityGross, liquidityNet)` into a tick slot +0 word. +fn pack_tick_word(gross: u128, net: i128) -> U256 { + U256::from(gross) | (U256::from(net as u128) << 128) +} + +/// Convert a `sol!`-decoded int24 tick to `i32` (a tick always fits in i24 ⊂ i32). +fn tick_to_i32(tick: alloy_primitives::aliases::I24) -> i32 { + i128::try_from(tick).unwrap_or(0) as i32 +} + +/// The pre-state context a `Mint`/`Burn` maintenance pass reads against: the pool +/// address, its layout, and the read-only [`StateView`]. +struct LiquidityCtx<'a> { + pool: Address, + layout: &'a UniswapV3Layout, + view: &'a dyn StateView, +} + +impl LiquidityCtx<'_> { + /// Maintenance for one tick endpoint of a `Mint`/`Burn`. `is_burn` selects the + /// sign (mint adds, burn subtracts); `is_lower` selects the `liquidityNet` + /// sign convention (lower += / upper -= on a mint). Appends the recomputed + /// tick-word write plus any `initialized`/bitmap flips (or a cold marker) to + /// `out`. + fn maintain_tick( + &self, + tick: i32, + amount: u128, + is_burn: bool, + is_lower: bool, + out: &mut Vec, + ) { + let keys = v3_tick_info_storage_keys_with_base(tick, self.layout.ticks_base_slot); + let base = keys[0]; + let slot3 = keys[3]; + + // The current packed tick word. Cold → cannot recompute: surface a marker. + let Some(word) = self.view.storage(self.pool, base) else { + out.push(cold_marker(self.pool, base)); + return; + }; + let (gross, net) = unpack_tick_word(word); + + // gross is always +amount on mint, -amount on burn (saturating defensively). + let new_gross = if is_burn { + gross.saturating_sub(amount) + } else { + gross.saturating_add(amount) + }; + // net: lower += amount, upper -= amount on mint; inverse on burn. + let net_delta = amount as i128; + let signed_delta = match (is_burn, is_lower) { + (false, true) => net_delta, // mint lower: + + (false, false) => -net_delta, // mint upper: - + (true, true) => -net_delta, // burn lower: - + (true, false) => net_delta, // burn upper: + + }; + let new_net = net.wrapping_add(signed_delta); + + out.push(StateUpdate::slot( + self.pool, + base, + pack_tick_word(new_gross, new_net), + )); + + // initialized flag (+3, bit 248) + bitmap bit flip on a 0↔positive cross. + let init_mask = U256::from(1) << TICK_INITIALIZED_BIT; + let newly_initialized = gross == 0 && new_gross > 0; + let now_uninitialized = gross > 0 && new_gross == 0; + + if newly_initialized { + out.push(StateUpdate::slot_masked( + self.pool, slot3, init_mask, init_mask, + )); + if let Some(flip) = self.bitmap_flip(tick, true) { + out.push(flip); + } + } else if now_uninitialized { + out.push(StateUpdate::slot_masked( + self.pool, + slot3, + init_mask, + U256::ZERO, + )); + if let Some(flip) = self.bitmap_flip(tick, false) { + out.push(flip); + } + } + } + + /// Build the `tickBitmap` word/bit flip for `tick` (`set` = newly initialized, + /// clear = newly uninitialized). The bitmap is keyed by the compressed tick + /// `tick / tick_spacing` (V3 guarantees `tick % tick_spacing == 0`, so the + /// division is exact). Returns `None` if `tick_spacing` is non-positive + /// (degenerate layout). + fn bitmap_flip(&self, tick: i32, set: bool) -> Option { + if self.layout.tick_spacing <= 0 { + return None; + } + let compressed = tick / self.layout.tick_spacing; + let word_pos = (compressed >> 8) as i16; + let bit_pos = (compressed & 0xFF) as u8; + let key = v3_tick_bitmap_storage_key_with_base(word_pos, self.layout.tick_bitmap_base_slot); + let mask = U256::from(1) << bit_pos; + let value = if set { mask } else { U256::ZERO }; + Some(StateUpdate::slot_masked(self.pool, key, mask, value)) + } + + /// Global-liquidity maintenance for a `Mint`/`Burn`: if the current `slot0` + /// tick is within `[tickLower, tickUpper)`, emit an absolute `liquidity` write + /// of `current ± amount`. Reads `slot0` and `liquidity` through the view; if + /// either is cold, surface a cold marker on the liquidity slot. + fn maintain_global_liquidity( + &self, + tick_lower: i32, + tick_upper: i32, + amount: u128, + is_burn: bool, + out: &mut Vec, + ) { + let liquidity_slot = self.layout.liquidity_slot; + let Some(slot0) = self.view.storage(self.pool, self.layout.slot0_slot) else { + out.push(cold_marker(self.pool, liquidity_slot)); + return; + }; + let current_tick = extract_tick(slot0); + if !(tick_lower <= current_tick && current_tick < tick_upper) { + return; // out of range: global liquidity unchanged. + } + let Some(current_word) = self.view.storage(self.pool, liquidity_slot) else { + out.push(cold_marker(self.pool, liquidity_slot)); + return; + }; + let current = u128::try_from(current_word & U256::from(u128::MAX)).unwrap_or(0); + let new = if is_burn { + current.saturating_sub(amount) + } else { + current.saturating_add(amount) + }; + out.push(StateUpdate::slot( + self.pool, + liquidity_slot, + U256::from(new), + )); + } + + /// Decode a `Mint` or `Burn` (shared tick + liquidity maintenance). + fn decode_liquidity_event( + &self, + tick_lower: i32, + tick_upper: i32, + amount: u128, + is_burn: bool, + ) -> Vec { + let mut out = Vec::new(); + self.maintain_tick(tick_lower, amount, is_burn, true, &mut out); + self.maintain_tick(tick_upper, amount, is_burn, false, &mut out); + self.maintain_global_liquidity(tick_lower, tick_upper, amount, is_burn, &mut out); + out + } +} + +/// Sign-extend the int24 `tick` field (bits [160,184)) out of a packed `slot0`. +fn extract_tick(slot0: U256) -> i32 { + let raw = ((slot0 >> SLOT0_TICK_SHIFT) & U256::from(0x00FF_FFFFu32)).to::(); + // Sign-extend from 24 bits. + if raw & 0x0080_0000 != 0 { + (raw | 0xFF00_0000) as i32 + } else { + raw as i32 + } +} + +impl EventDecoder for UniswapV3Decoder { + fn decode(&self, log: &Log, view: &dyn StateView) -> Vec { + let Some(layout) = self.pools.get(&log.address) else { + return Vec::new(); + }; + let pool = log.address; + let topic0 = match log.topics().first() { + Some(t) => *t, + None => return Vec::new(), + }; + + if topic0 == Swap::SIGNATURE_HASH { + let Ok(swap) = Swap::decode_log_data(&log.data) else { + return Vec::new(); + }; + // slot0: masked write of sqrtPriceX96 [0,160) + tick [160,184), + // preserving observation / feeProtocol / unlocked bits [184,256). + let mask = (U256::from(1) << SLOT0_PRICE_TICK_BITS) - U256::from(1); + let sqrt_price = U256::from_be_slice(swap.sqrtPriceX96.to_be_bytes::<20>().as_slice()); + let tick = tick_to_i32(swap.tick); + let tick24 = U256::from((tick as u32) & 0x00FF_FFFF); + let value = sqrt_price | (tick24 << SLOT0_TICK_SHIFT); + vec![ + StateUpdate::slot_masked(pool, layout.slot0_slot, mask, value), + // liquidity: absolute (the event carries post-swap liquidity). + StateUpdate::slot(pool, layout.liquidity_slot, U256::from(swap.liquidity)), + ] + } else if topic0 == Mint::SIGNATURE_HASH { + let Ok(mint) = Mint::decode_log_data(&log.data) else { + return Vec::new(); + }; + let ctx = LiquidityCtx { pool, layout, view }; + ctx.decode_liquidity_event( + tick_to_i32(mint.tickLower), + tick_to_i32(mint.tickUpper), + mint.amount, + false, + ) + } else if topic0 == Burn::SIGNATURE_HASH { + let Ok(burn) = Burn::decode_log_data(&log.data) else { + return Vec::new(); + }; + let ctx = LiquidityCtx { pool, layout, view }; + ctx.decode_liquidity_event( + tick_to_i32(burn.tickLower), + tick_to_i32(burn.tickUpper), + burn.amount, + true, + ) + } else { + Vec::new() + } + } +} diff --git a/src/lib.rs b/src/lib.rs index bc20262..15406b8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -46,9 +46,14 @@ //! policy, mechanism) and the optimistic verify-and-rerun execution loop with //! deferred validation. //! - [`state_update`] — the generic state-mutation vocabulary (`StateUpdate` / -//! `AccountPatch` / `PurgeScope`, plus relative `SlotDelta` read-modify-write) -//! applied by `EvmCache::apply_update` / `apply_updates` / `modify_slot`, with a -//! structured `StateDiff` output (Pillar B.1). +//! `AccountPatch` / `PurgeScope`, plus relative `SlotDelta` read-modify-write and +//! masked `SlotMasked` writes) applied by `EvmCache::apply_update` / +//! `apply_updates` / `modify_slot`, with a structured `StateDiff` output +//! (Pillar B.1). +//! - [`events`] — the event → state pipeline (Pillar B.2): `EventDecoder` / +//! `StateView` / `DecoderRegistry` decode an on-chain `Log` into `StateUpdate`s, +//! and `EventPipeline` ingests, reorg-purges, and reconciles a block's logs. +//! Ships an ERC-20 `Transfer` decoder and (under `protocols`) a UniswapV3 adapter. //! - [`inspector`] — an [`Inspector`](revm::Inspector) that captures ERC20 //! `Transfer` events to reconstruct balance deltas from a simulation. //! - [`multicall`] — batched read-only calls through Multicall3. @@ -102,6 +107,7 @@ pub mod cache; pub mod create3; pub mod deploy; pub mod errors; +pub mod events; pub mod freshness; pub mod inspector; pub mod multicall; @@ -109,6 +115,13 @@ pub mod prefetch_registry; pub mod state_update; pub use access_set::StorageAccessList; +pub use events::erc20::Erc20TransferDecoder; +#[cfg(feature = "protocols")] +pub use events::uniswap_v3::{UniswapV3Decoder, UniswapV3Layout}; +pub use events::{ + BlockDigest, DecoderRegistry, EventDecoder, EventPipeline, ReconcileReport, ReorgConfig, + StateView, +}; pub use freshness::{ AlwaysVerify, BlockClock, FreshnessClock, FreshnessController, FreshnessParams, FreshnessPolicy, FreshnessRegistry, NeverVerify, ObservationDriven, SimRequest, SlotChange, diff --git a/tests/event_pipeline.rs b/tests/event_pipeline.rs index bef20d7..1905f30 100644 --- a/tests/event_pipeline.rs +++ b/tests/event_pipeline.rs @@ -609,6 +609,7 @@ mod uniswap_v3 { U256::from(sqrt_price) | (tick24 << 160) | (high << 184) } + #[allow(dead_code)] fn tick_word(gross: u128, net: i128) -> U256 { U256::from(gross) | (U256::from(net as u128) << 128) } From 0749dfbf6d96e6643cbfb5201f8526df6c68bfc2 Mon Sep 17 00:00:00 2001 From: Kai Aldag Date: Tue, 16 Jun 2026 13:39:46 +0100 Subject: [PATCH 18/26] Phase 4: offline example, benchmark, docs (overseer deliverables) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the Phase 4 deliverable on top of the sub-agent's src/ implementation: - examples/reactive_cache.rs (offline): register an ERC-20 + UniswapV3 decoder, ingest a block of logs (a Transfer + a Swap), show the BlockDigest, the preserved slot0 `unlocked` bit, a reconcile drift alarm+correction, and a reorg purge. Feature-gated (UniswapV3 adapter) with a fallback main. - benches/event_pipeline.rs (offline, registered in Cargo.toml): per-event decode cost (ERC-20 Transfer ~435ns, V3 Swap ~68ns, V3 Mint ~1.0us), ingest_logs decode+apply throughput (linear, ~112ns/log to 1000), reorg_to purge cost (~378us/1000 addrs). - CHANGELOG: Phase 4 `### Added` (event pipeline + adapters + SlotMasked). - ROADMAP: Phase 4 row -> Done + a "Landed on ..." section. - KNOWN_ISSUES: refreshed the Pillar B status (reader+writer halves done, live WS transport not) and recorded the §6.4 V3 fee-growth/oracle maintenance gap. - README: reactive_cache + event_pipeline rows. - tests/event_pipeline.rs: removed an unused tick_word helper (mine; the sub-agent had `#[allow(dead_code)]`-silenced it) + fmt. Independently verified green on both feature configs: fmt; clippy --all-targets (default) + --lib --no-default-features; cargo test (321 passed = 289 tests + 32 doctests) + --no-default-features (269 = 241 + 28); RUSTDOCFLAGS=-D warnings doc; cargo bench --no-run. The example runs offline. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 36 ++++++ Cargo.toml | 4 + README.md | 2 + benches/event_pipeline.rs | 240 ++++++++++++++++++++++++++++++++++++ docs/KNOWN_ISSUES.md | 26 +++- docs/ROADMAP.md | 52 +++++++- examples/reactive_cache.rs | 243 +++++++++++++++++++++++++++++++++++++ tests/event_pipeline.rs | 4 - 8 files changed, 597 insertions(+), 10 deletions(-) create mode 100644 benches/event_pipeline.rs create mode 100644 examples/reactive_cache.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index bb58ca9..0968cf3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,42 @@ pre-release development phases (see [`docs/ROADMAP.md`](docs/ROADMAP.md)). and the `SlotDelta` double-read of the old value is eliminated. The result is byte-identical to folding `apply_update` over the batch (pinned by the batched==sequential equivalence test). Generic core. +- **Event → state pipeline** (`events` module, Phase 4, Pillar B.2 — the *reader + half* of the event pipeline) — turn on-chain logs into the Phase 3 `StateUpdate` + vocabulary and drive them through the cache for reactive freshness: + - **`EventDecoder` / `StateView`** — a decoder is a pure function of + `(log, pre-state)` returning `Vec`; the narrow read-only + `StateView` (implemented by `EvmCache` via `cached_storage_value`) lets + stateful adapters read current cached state without RPC. Generic core. + - **`DecoderRegistry`** — dispatches a log to the decoders registered for its + emitting address (plus globals) and concatenates their output. Generic core. + - **`Erc20TransferDecoder`** — decodes ERC-20 `Transfer` logs into relative + balance `SlotDelta`s (skipping the zero-address mint/burn leg), with per-token + balance-slot config. The reactive-balance case from Phase 3 §15, now + log-driven. Generic core. + - **`UniswapV3Decoder` / `UniswapV3Layout`** (`protocols`) — `Swap` → a masked + `slot0` write (new `sqrtPriceX96` + `tick`, **preserving** the + observation/fee/`unlocked` bits — a clobbered `unlocked` would make a quote + revert `LOK`) plus an absolute `liquidity` write; `Mint`/`Burn` → per-tick + `liquidityGross`/`liquidityNet`, the `initialized` flag, the `tickBitmap` + word bit, and the in-range global `liquidity`, computed against the + `StateView` and cold-aware. Uniswap and PancakeSwap layouts. + - **`EventPipeline`** — `ingest_logs` decodes + applies a block's logs + **log-by-log in order** (so a later log sees earlier applies) and returns a + `BlockDigest`; `reorg_to` purges (purge-and-resync) the addresses touched + after a new head; `reconcile` re-reads sampled event-derived slots against + chain truth (correct **and** alarm) via the new `EvmCache::reconcile_slots`. A + thin async `drive`/`LogSource` convenience layers the synchronous core over a + stream. Generic core. +- **`StateUpdate::SlotMasked`** (`state_update`, Phase 4) — a cold-aware + read-modify-write *masked* slot write (`new = (old & !mask) | (value & mask)`) + with the `StateUpdate::slot_masked` constructor, so a pure decoder can update + selected bits of a **packed** storage word (e.g. V3 `slot0`) without clobbering + the rest. A masked write to a cold slot is skipped and surfaced in the new + `StateDiff.skipped_masks: Vec` (counted by `has_skipped` / + `skipped_len`, not by the changes-only `is_empty`/`len`); `serde` on + `SkippedMask`. Adding the variant and the field is permitted under the pre-1.0 + break policy (`StateUpdate`/`StateDiff` are `#[non_exhaustive]`). Generic core. - **Configurable transaction & block environment** — `TxConfig` (value, gas limit, gas price, nonce, access list) threaded through `call_raw_with`; block context setters (`set_coinbase`, `set_prevrandao`, `set_block_gas_limit`). diff --git a/Cargo.toml b/Cargo.toml index cdde9d5..4a928d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -90,6 +90,10 @@ harness = false name = "state_update" harness = false +[[bench]] +name = "event_pipeline" +harness = false + # RPC-gated real-contract benchmarks. Skipped (not failed) when RPC_URL is unset, # so `cargo bench` stays offline by default. [[bench]] diff --git a/README.md b/README.md index 0d1b959..f106255 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,7 @@ and inject all state directly: | `freshness_optimistic` | Advanced | Optimistic verify-and-rerun loop: a `Corrected` validation via a stub fetcher. | | `freshness_multi_sim` | Advanced | Many sims with selective re-run, plus classification and `ValidThrough` aging. | | `state_update_apply` | Advanced | Apply a mixed `StateUpdate` batch (`Slot`/`Account`/`Purge`) and inspect the returned `StateDiff`. | +| `reactive_cache` | Advanced | Decode logs (ERC-20 `Transfer` + UniswapV3 `Swap`) into `StateUpdate`s, ingest a block, reconcile drift, and purge on a reorg. | **RPC examples** fork real mainnet state. Set `RPC_URL` to an Ethereum RPC endpoint (they print instructions and exit if it is unset): @@ -226,6 +227,7 @@ cache sizes: | `simulation` | `create_snapshot` across cache sizes (100 → 10k accounts), overlay fan-out, `call_raw` throughput, sequential bundle execution, batched storage injection. | | `freshness` | The optimistic loop end-to-end (CPU and latency-hiding), `verify_slots` at scale (1 → 1000 slots), and multi-sim fan-out. | | `state_update` | `apply_updates` throughput across batch sizes (1 → 1000 `Slot`s) and per-variant apply cost (`Slot` vs `Account` vs `Purge`). | +| `event_pipeline` | Per-event decode cost (ERC-20 `Transfer`, V3 `Swap`/`Mint`), `ingest_logs` decode+apply throughput (1 → 1000 logs), and `reorg_to` purge cost. | | `access_list` | Touch-set merge and EIP-2930 list construction. | | `revert_decoding` | Built-in and custom revert decoding, including decoder dispatch with many registered errors. | | `storage_keys` | Mapping/array storage-key derivation. | diff --git a/benches/event_pipeline.rs b/benches/event_pipeline.rs new file mode 100644 index 0000000..9ddf859 --- /dev/null +++ b/benches/event_pipeline.rs @@ -0,0 +1,240 @@ +//! Phase 4 benchmarks: the event → state pipeline (Pillar B.2). +//! +//! Measures three things, all offline (mocked provider, in-memory logs): +//! - **decode** cost per event kind (ERC-20 `Transfer`, UniswapV3 `Swap`/`Mint`), +//! isolating the pure `EventDecoder::decode` work (no apply); +//! - **ingest** throughput — [`EventPipeline::ingest_logs`] decoding **and** +//! applying a block of logs, across batch sizes (1 → 1000); +//! - **reorg** purge cost — [`EventPipeline::reorg_to`] over a touched set of +//! 1 → 1000 addresses. +//! +//! A current-thread runtime drives only the async cache constructor; the pipeline +//! itself is synchronous and never touches the network. + +use std::collections::HashMap; +use std::hint::black_box; +use std::sync::Arc; + +use alloy_primitives::aliases::{I24, U160}; +use alloy_primitives::{Address, Bytes, I256, Log, U256, hex, keccak256}; +use alloy_provider::RootProvider; +use alloy_provider::network::AnyNetwork; +use alloy_rpc_client::RpcClient; +use alloy_sol_types::{SolEvent, sol}; +use alloy_transport::mock::Asserter; +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use evm_fork_cache::cache::{EvmCache, V3_SLOT0_SLOT, v3_tick_info_storage_keys_with_base}; +use evm_fork_cache::events::{DecoderRegistry, EventDecoder, EventPipeline, StateView}; +use evm_fork_cache::{Erc20TransferDecoder, StateUpdate, UniswapV3Decoder, UniswapV3Layout}; +use revm::state::{AccountInfo, Bytecode}; +use tokio::runtime::{Builder, Runtime}; + +const MOCK_ERC20_RUNTIME_HEX: &str = include_str!("../fixtures/mock_erc20_runtime.hex"); +const TOKEN: Address = Address::repeat_byte(0xAA); +const POOL: Address = Address::repeat_byte(0xBB); + +fn current_thread_rt() -> Runtime { + Builder::new_current_thread().enable_all().build().unwrap() +} + +/// A cache with `TOKEN` and `POOL` installed as storage-cleared accounts (so +/// unseeded slots read as zero — no RPC fallthrough). +fn seeded_cache(rt: &Runtime) -> EvmCache { + let provider = RootProvider::::new(RpcClient::mocked(Asserter::new())); + let mut cache = rt.block_on(EvmCache::new(Arc::new(provider), None)); + let runtime = Bytecode::new_raw(Bytes::from( + hex::decode(MOCK_ERC20_RUNTIME_HEX.trim()).unwrap(), + )); + let code_hash = runtime.hash_slow(); + for addr in [TOKEN, POOL] { + cache.db_mut().insert_account_info( + addr, + AccountInfo { + balance: U256::ZERO, + nonce: 0, + code: Some(runtime.clone()), + code_hash, + account_id: None, + }, + ); + cache + .db_mut() + .replace_account_storage(addr, Default::default()) + .unwrap(); + } + cache +} + +sol! { + event Swap(address indexed sender, address indexed recipient, int256 amount0, int256 amount1, uint160 sqrtPriceX96, uint128 liquidity, int24 tick); + event Mint(address sender, address indexed owner, int24 indexed tickLower, int24 indexed tickUpper, uint128 amount, uint256 amount0, uint256 amount1); +} + +fn transfer_log(token: Address, from: Address, to: Address, value: U256) -> Log { + let sig = keccak256(b"Transfer(address,address,uint256)"); + Log::new_unchecked( + token, + vec![sig, from.into_word(), to.into_word()], + Bytes::copy_from_slice(&value.to_be_bytes::<32>()), + ) +} + +fn swap_log(pool: Address, sqrt_price: u128, liquidity: u128, tick: i32) -> Log { + let ev = Swap { + sender: Address::repeat_byte(0x01), + recipient: Address::repeat_byte(0x02), + amount0: I256::try_from(-1i64).unwrap(), + amount1: I256::try_from(1i64).unwrap(), + sqrtPriceX96: U160::from(sqrt_price), + liquidity, + tick: I24::try_from(tick).unwrap(), + }; + Log { + address: pool, + data: ev.encode_log_data(), + } +} + +fn mint_log(pool: Address, lower: i32, upper: i32, amount: u128) -> Log { + let ev = Mint { + sender: Address::repeat_byte(0x03), + owner: Address::repeat_byte(0x04), + tickLower: I24::try_from(lower).unwrap(), + tickUpper: I24::try_from(upper).unwrap(), + amount, + amount0: U256::from(1), + amount1: U256::from(1), + }; + Log { + address: pool, + data: ev.encode_log_data(), + } +} + +/// A bench-local read-only [`StateView`] over a fixed map (for the V3 `Mint` +/// decode, which reads the current tick word). +struct MapView(HashMap<(Address, U256), U256>); +impl StateView for MapView { + fn storage(&self, address: Address, slot: U256) -> Option { + self.0.get(&(address, slot)).copied() + } +} + +/// A bench-local decoder that emits one absolute `Slot` write per log, keyed by +/// the log's address — so repeated ingest is idempotent (stable across iters). +struct AbsDecoder; +impl EventDecoder for AbsDecoder { + fn decode(&self, log: &Log, _view: &dyn StateView) -> Vec { + vec![StateUpdate::slot(log.address, U256::from(0), U256::from(1))] + } +} + +/// Pure `decode` cost per event kind (no apply). +fn bench_decode(c: &mut Criterion) { + let mut group = c.benchmark_group("decode"); + + let erc20 = Erc20TransferDecoder::new(U256::from(3)); + let tlog = transfer_log( + TOKEN, + Address::repeat_byte(0x21), + Address::repeat_byte(0x22), + U256::from(100), + ); + let empty = MapView(HashMap::new()); + group.bench_function("erc20_transfer", |b| { + b.iter(|| black_box(erc20.decode(black_box(&tlog), &empty))) + }); + + let v3 = UniswapV3Decoder::new().with_pool(POOL, UniswapV3Layout::uniswap(60)); + let slog = swap_log(POOL, 2_000_000, 7_500, 120); + let mut slot0_view = HashMap::new(); + slot0_view.insert( + (POOL, V3_SLOT0_SLOT), + (U256::from(1u64) << 240) | U256::from(1_000_000u64), + ); + // Seed the tick words the Mint reads (lower/upper) so it computes (not skips). + let lo = v3_tick_info_storage_keys_with_base(60, evm_fork_cache::cache::V3_TICKS_BASE_SLOT)[0]; + let hi = v3_tick_info_storage_keys_with_base(120, evm_fork_cache::cache::V3_TICKS_BASE_SLOT)[0]; + slot0_view.insert((POOL, lo), U256::ZERO); + slot0_view.insert((POOL, hi), U256::ZERO); + let view = MapView(slot0_view); + group.bench_function("v3_swap", |b| { + b.iter(|| black_box(v3.decode(black_box(&slog), &view))) + }); + let mlog = mint_log(POOL, 60, 120, 1_000); + group.bench_function("v3_mint", |b| { + b.iter(|| black_box(v3.decode(black_box(&mlog), &view))) + }); + + group.finish(); +} + +/// `ingest_logs` decode+apply throughput as the per-block log batch grows. +fn bench_ingest_batch(c: &mut Criterion) { + let rt = current_thread_rt(); + let mut cache = seeded_cache(&rt); + + let mut group = c.benchmark_group("ingest_logs"); + for &n in &[1usize, 10, 100, 1_000] { + let logs: Vec = (0..n) + .map(|i| { + Log::new_unchecked( + Address::repeat_byte((i % 251 + 1) as u8), + vec![], + Bytes::new(), + ) + }) + .collect(); + let mut registry = DecoderRegistry::new(); + registry.register(Arc::new(AbsDecoder)); + let mut pipeline = EventPipeline::new(registry); + + group.throughput(Throughput::Elements(n as u64)); + group.bench_with_input(BenchmarkId::from_parameter(n), &logs, |b, logs| { + let mut block = 0u64; + b.iter(|| { + block += 1; + black_box(pipeline.ingest_logs(&mut cache, block, black_box(logs))) + }) + }); + } + group.finish(); +} + +/// `reorg_to` purge cost over a touched set of N distinct addresses. +fn bench_reorg(c: &mut Criterion) { + let rt = current_thread_rt(); + + let mut group = c.benchmark_group("reorg_to"); + for &n in &[10usize, 100, 1_000] { + group.throughput(Throughput::Elements(n as u64)); + group.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, &n| { + b.iter_batched( + || { + // Setup: a cache + pipeline with N addresses touched at block 1. + let mut cache = seeded_cache(&rt); + let mut registry = DecoderRegistry::new(); + registry.register(Arc::new(AbsDecoder)); + let mut pipeline = EventPipeline::new(registry); + let logs: Vec = (0..n) + .map(|i| { + let mut bytes = [0u8; 20]; + bytes[0..8].copy_from_slice(&(i as u64).to_be_bytes()); + Log::new_unchecked(Address::from(bytes), vec![], Bytes::new()) + }) + .collect(); + pipeline.ingest_logs(&mut cache, 1, &logs); + (pipeline, cache) + }, + |(mut pipeline, mut cache)| { + black_box(pipeline.reorg_to(&mut cache, 0)); + }, + criterion::BatchSize::SmallInput, + ) + }); + } + group.finish(); +} + +criterion_group!(benches, bench_decode, bench_ingest_batch, bench_reorg); +criterion_main!(benches); diff --git a/docs/KNOWN_ISSUES.md b/docs/KNOWN_ISSUES.md index 06a755d..ce24b7c 100644 --- a/docs/KNOWN_ISSUES.md +++ b/docs/KNOWN_ISSUES.md @@ -139,11 +139,27 @@ Confidence legend: **[V]** verified against the source during review; unit tests assume the default feature. Extraction into `evm-amm-state` is planned (roadmap), blocked partly by `ImmutableDataCache` coupling generic token-decimals with V2/V3/Balancer pool metadata. -- **Event-driven sync (roadmap Pillar B) is not implemented.** Targeted - inject/purge exist; the Phase 3 **writer half** (the `StateUpdate` vocabulary + - `apply_update`/`apply_updates`) lands the apply mechanism, but decoding logs - into state updates and the WS ingestion loop with reorg handling are future - phases (Pillar B.2). +- **Event-driven sync (roadmap Pillar B) — reader/writer halves done; live WS + transport is not.** The Phase 3 **writer half** (`StateUpdate` + + `apply_update`/`apply_updates`) and the Phase 4 **reader half** (the `events` + module: `EventDecoder`/`DecoderRegistry`, the ERC-20 + UniswapV3 adapters, and + the `EventPipeline` with `ingest_logs`/`reorg_to`/`reconcile`) are implemented. + What is **not** shipped is a concrete production WS transport: the async + `events::drive`/`LogSource` convenience is generic over a log source and is + exercised only by the offline example feeding an in-memory source; wiring it to + a live `subscribe_logs`/WS provider (and detecting reorgs from block-hash + mismatches) is left to the consumer. +- **[V] V3 event-derived tick maintenance does not reconstruct fee-growth / + oracle state (Phase 4 §6.4).** `UniswapV3Decoder`'s `Mint`/`Burn` handling + maintains `liquidityGross`/`liquidityNet` (tick slot +0), the `initialized` flag + (+3), the `tickBitmap`, and the in-range global `liquidity`, but **not** + `feeGrowthOutside0/1X128` (slots +1/+2), `secondsOutside`, or oracle + observations — these are not derivable from the `Mint`/`Burn`/`Swap` events. + **Swap price/liquidity quoting is unaffected** (the swap-amount math does not + read `feeGrowthOutside`), but fee accounting and `collect`-style reads against + event-maintained ticks are not kept current. Sampled + `EventPipeline::reconcile` (RPC re-read) and reorg `reorg_to` (purge-and-resync) + are the backstop; seed a full tick via `inject_v3_ticks` when fee state matters. - **`inject_v2/v3_*` layer behavior changed in Phase 3 (Decision 2).** The `protocols`-gated `inject_v2_pool_metadata` / `inject_v3_tick_bitmap*` / `inject_v3_ticks*` helpers were refolded onto the write-through diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index e192c15..8eaa262 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -73,7 +73,7 @@ RPC node Event-driven sync ← WS logs · new block | **1** | Engine seam: typed errors, configurable tx/block env, hot-path benches, builder, `protocols` feature. | **Done** (`phase-1-engine-seam`) | | **2** | Freshness core (Pillar C): `Validity` + `FreshnessRegistry`; observation tracker; policies; optimistic verify-and-rerun loop. | **Done** (`phase-2-freshness`) | | **3** | State-update primitives (Pillar B.1): `StateUpdate` + targeted writers; refold `inject_*`; surface state-diff output. | **Done** (`phase-3-state-updates`) | -| **4** | Event pipeline + adapters (Pillar B.2): `EventDecoder` trait, V3 adapter, WS ingestion loop, reorg handling. | Planned | +| **4** | Event pipeline + adapters (Pillar B.2): `EventDecoder` trait, ERC-20 + V3 adapters, ingest/reorg/reconcile pipeline. | **Done** (`phase-4-event-pipeline`) | | **5** | COW snapshots (Pillar A): structural sharing; overlay buffer reuse. | Planned | Cross-cutting (land opportunistically): call tracer Inspector, full offline @@ -356,6 +356,56 @@ equivalence test). --- +## Phase 4 — event pipeline + adapters (detailed, decisions locked) + +Builds **Pillar B.2 — the reader half** of the event → state pipeline: decode an +on-chain `Log` into the Phase 3 `StateUpdate` vocabulary, apply it, and run the +reactive maintenance (reconcile, reorg) that keeps event-derived state honest. +Decoders are pure functions of `(log, pre-state)`; the `!Send` cache discipline is +preserved by keeping the tested core synchronous (the async ingestion driver is a +thin convenience). The full build contract is in +[`phase-4-spec.md`](phase-4-spec.md). + +### Locked decisions + +1. **Packed-slot updates → `StateUpdate::SlotMasked`** (a cold-aware RMW masked + write), so a pure decoder can express a partial update to a packed word (V3 + `slot0`) without clobbering the bits it does not own (notably `unlocked`). +2. **V3 adapter coverage → `Swap` **and** `Mint`/`Burn` (full ticks).** `slot0` + + `liquidity` from `Swap`; per-tick `liquidityGross`/`liquidityNet` + + `initialized` + `tickBitmap` + in-range global `liquidity` from `Mint`/`Burn`, + computed against the `StateView`. Fee-growth/oracle state is out of scope (a + documented limitation; reconcile/purge are the backstop). +3. **Reorg → purge-and-resync.** A depth-bounded ring tracks addresses touched per + block; `reorg_to(n)` purges everything touched after `n` so reads re-fetch. + `ValidThrough` is the freshness lever. +4. **Reconciliation → sampled re-read, correct **and** alarm.** `reconcile` samples + event-derived slots and re-reads via `EvmCache::reconcile_slots` (a honest + wrapper over `verify_slots` that errors on a total fetch failure rather than + reporting a false all-clear); the fresh chain value wins and the drift is + surfaced. + +### Acceptance — met + +`cargo fmt --check`, `clippy --all-targets -- -D warnings` (default + +`--lib --no-default-features`), `cargo test` (both feature configs), +`RUSTDOCFLAGS=-D warnings cargo doc`, `cargo bench --no-run`. + +Landed on `phase-4-event-pipeline`: `src/events/` (the generic core — +`EventDecoder`/`StateView`, `DecoderRegistry`, `EventPipeline` with +`ingest_logs`/`reorg_to`/`reconcile` + `BlockDigest`/`ReconcileReport`/ +`ReorgConfig`, and the async `drive`/`LogSource`), the generic +`Erc20TransferDecoder` (`events::erc20`), and the `protocols`-gated +`UniswapV3Decoder`/`UniswapV3Layout` (`events::uniswap_v3`); the cold-aware +`StateUpdate::SlotMasked` vocabulary + `StateDiff.skipped_masks`/`SkippedMask` +(`state_update`) and its dual-layer apply arm; `EvmCache::reconcile_slots` and the +`StateView` impl; the offline `examples/reactive_cache.rs`; +`benches/event_pipeline.rs`; and `tests/event_pipeline.rs` (+ the `SlotMasked` +tests in `tests/state_update.rs`). The §6.4 V3 fee-growth/oracle maintenance gap +is recorded in `KNOWN_ISSUES.md`. + +--- + ## Key abstractions for later phases (sketches) ```rust diff --git a/examples/reactive_cache.rs b/examples/reactive_cache.rs new file mode 100644 index 0000000..7901f83 --- /dev/null +++ b/examples/reactive_cache.rs @@ -0,0 +1,243 @@ +//! Reactive cache updates from the event stream (Pillar B.2). +//! +//! Decodes on-chain logs into the Phase 3 [`StateUpdate`] vocabulary and applies +//! them to a fork cache — keeping hot state fresh **without** an RPC round-trip +//! per change. It wires up the three pieces Phase 4 adds: +//! +//! 1. A [`DecoderRegistry`] with an [`Erc20TransferDecoder`] (balances) and a +//! [`UniswapV3Decoder`] (a pool's `slot0` price/tick + `liquidity`). +//! 2. An [`EventPipeline`] whose `ingest_logs` decodes + applies a block's logs +//! (log-by-log), surfacing a [`BlockDigest`]. +//! 3. The reactive maintenance: the freshness wiring (pin event-derived slots so +//! the optimistic validator does not re-verify them, then advance the block +//! clock), a sampled **reconcile** drift alarm against a stub fetcher, and a +//! **reorg** purge-and-resync. +//! +//! Runs fully offline against a mocked provider and in-memory logs — no network. +//! Requires the `protocols` feature (the UniswapV3 adapter), which is on by +//! default. +//! +//! Run with: +//! +//! ```sh +//! cargo run --example reactive_cache +//! ``` + +#[cfg(feature = "protocols")] +#[tokio::main(flavor = "multi_thread")] +async fn main() -> anyhow::Result<()> { + imp::run().await +} + +#[cfg(not(feature = "protocols"))] +fn main() { + eprintln!( + "the `reactive_cache` example requires the `protocols` feature (the \ + UniswapV3 adapter). Run it with default features: \ + `cargo run --example reactive_cache`." + ); +} + +#[cfg(feature = "protocols")] +#[path = "support/mock.rs"] +mod mock; + +#[cfg(feature = "protocols")] +mod imp { + use std::collections::HashMap; + use std::sync::Arc; + + use alloy_eips::BlockId; + use alloy_primitives::aliases::{I24, U160}; + use alloy_primitives::{Address, Bytes, I256, Log, U256, keccak256}; + use alloy_sol_types::{SolEvent, SolValue, sol}; + use anyhow::Result; + use evm_fork_cache::cache::{StorageBatchFetchFn, V3_LIQUIDITY_SLOT, V3_SLOT0_SLOT}; + use evm_fork_cache::events::{DecoderRegistry, EventPipeline}; + use evm_fork_cache::freshness::{ + AlwaysVerify, FreshnessController, FreshnessRegistry, Validity, + }; + use evm_fork_cache::{Erc20TransferDecoder, UniswapV3Decoder, UniswapV3Layout}; + + use super::mock; + + sol! { + event Swap(address indexed sender, address indexed recipient, int256 amount0, int256 amount1, uint160 sqrtPriceX96, uint128 liquidity, int24 tick); + } + + /// Hashed `balanceOf[owner]` slot for the MockERC20 fixture (mapping at slot 3). + fn balance_slot(owner: Address) -> U256 { + let key = keccak256((owner, U256::from(mock::MOCK_ERC20_BALANCE_SLOT)).abi_encode()); + U256::from_be_bytes(key.0) + } + + /// Build an ERC-20 `Transfer(from, to, value)` log. + fn transfer_log(token: Address, from: Address, to: Address, value: U256) -> Log { + let sig = keccak256(b"Transfer(address,address,uint256)"); + Log::new_unchecked( + token, + vec![sig, from.into_word(), to.into_word()], + Bytes::copy_from_slice(&value.to_be_bytes::<32>()), + ) + } + + /// Build a UniswapV3 `Swap` log carrying the post-swap price/liquidity/tick. + fn swap_log(pool: Address, sqrt_price: u128, liquidity: u128, tick: i32) -> Log { + let ev = Swap { + sender: Address::repeat_byte(0x5e), + recipient: Address::repeat_byte(0x5f), + amount0: I256::try_from(-1_000i64).unwrap(), + amount1: I256::try_from(1_000i64).unwrap(), + sqrtPriceX96: U160::from(sqrt_price), + liquidity, + tick: I24::try_from(tick).unwrap(), + }; + Log { + address: pool, + data: ev.encode_log_data(), + } + } + + /// Pack a slot0 word: sqrtPriceX96 [0,160), tick [160,184), `unlocked` at bit 240. + fn pack_slot0(sqrt_price: u128, tick: i32) -> U256 { + let tick24 = U256::from((tick as u32) & 0x00FF_FFFF); + let unlocked = U256::from(1) << 240; + U256::from(sqrt_price) | (tick24 << 160) | unlocked + } + + pub async fn run() -> Result<()> { + let mut cache = mock::offline_cache().await?; + + let token = Address::repeat_byte(0x11); + let pool = Address::repeat_byte(0x99); + let alice = Address::repeat_byte(0x22); + let bob = Address::repeat_byte(0x33); + mock::install_default_account(&mut cache, Address::ZERO); + mock::install_default_account(&mut cache, alice); + mock::install_default_account(&mut cache, bob); + mock::install_mock_erc20(&mut cache, token); + mock::install_mock_erc20(&mut cache, pool); // reuse as a storage-cleared pool + + // Seed the holders' balances (EVM-visible) and the pool's slot0 + liquidity. + cache + .db_mut() + .insert_account_storage(token, balance_slot(alice), U256::from(1_000))?; + cache + .db_mut() + .insert_account_storage(token, balance_slot(bob), U256::from(0))?; + cache + .db_mut() + .insert_account_storage(pool, V3_SLOT0_SLOT, pack_slot0(1_000_000, 100))?; + cache + .db_mut() + .insert_account_storage(pool, V3_LIQUIDITY_SLOT, U256::from(5_000))?; + + // 1. Build the decoder registry: ERC-20 balances + the V3 pool. + let mut registry = DecoderRegistry::new(); + registry.register(Arc::new(Erc20TransferDecoder::new(U256::from( + mock::MOCK_ERC20_BALANCE_SLOT, + )))); + registry.register(Arc::new( + UniswapV3Decoder::new().with_pool(pool, UniswapV3Layout::uniswap(60)), + )); + let mut pipeline = EventPipeline::new(registry); + + // The freshness side: a controller whose registry we pin event-derived + // slots into so the optimistic validator never re-verifies them by RPC. + let mut controller = FreshnessController::new(FreshnessRegistry::new(), AlwaysVerify); + + // 2. Ingest block 100: Alice sends Bob 250 tokens, and the pool swaps (new + // price/tick + liquidity). Decoded + applied log-by-log. + let block = 100u64; + let digest = pipeline.ingest_logs( + &mut cache, + block, + &[ + transfer_log(token, alice, bob, U256::from(250)), + swap_log(pool, 2_000_000, 7_500, 120), + ], + ); + + println!("=== ingested block {} ===", digest.block); + println!( + " decoded {} log(s) -> {} slot change(s), {} skipped", + digest.decoded_logs, + digest.applied.slots.len(), + digest.applied.skipped_len(), + ); + println!( + " alice balance: {} bob balance: {}", + mock::balance_of(&mut cache, token, alice)?, + mock::balance_of(&mut cache, token, bob)?, + ); + println!( + " pool liquidity slot: {}", + cache.cached_storage_value(pool, V3_LIQUIDITY_SLOT).unwrap() + ); + // slot0: the new price/tick landed, and the `unlocked` bit (240) is + // preserved by the masked write — a clobbered `unlocked` would make a + // quote revert LOK. + let slot0 = cache.cached_storage_value(pool, V3_SLOT0_SLOT).unwrap(); + println!( + " pool slot0 sqrtPriceX96 (low 160b): {}", + slot0 & ((U256::from(1) << 160) - U256::from(1)) + ); + println!( + " pool slot0 unlocked bit preserved: {}", + (slot0 >> 240) & U256::from(1) == U256::from(1) + ); + + // 3a. Freshness wiring: pin the touched slots (kept fresh out-of-band by + // the pipeline) and advance the block clock. + for (addr, slot) in &digest.touched_slots { + controller + .registry_mut() + .set_slot(*addr, *slot, Validity::Pinned); + } + controller.on_new_block(block); + println!( + "\npinned {} event-derived slot(s) into the freshness registry", + digest.touched_slots.len() + ); + + // 3b. Sampled reconcile against chain truth. Stub the fetcher so the + // pool's liquidity reads 7_600 on-chain (a small drift from our + // event-derived 7_500): reconcile corrects the cache AND alarms. + let fresh: HashMap<(Address, U256), U256> = + HashMap::from([((pool, V3_LIQUIDITY_SLOT), U256::from(7_600))]); + let fetcher: StorageBatchFetchFn = Arc::new( + move |requests: Vec<(Address, U256)>, _block: Option| { + requests + .into_iter() + .map(|(a, s)| (a, s, Ok(fresh.get(&(a, s)).copied().unwrap_or(U256::ZERO)))) + .collect() + }, + ); + cache.set_storage_batch_fetcher(fetcher); + + let report = pipeline.reconcile(&mut cache, &[(pool, V3_LIQUIDITY_SLOT)])?; + println!("\n=== reconcile (sampled {} slot) ===", report.checked); + if report.mismatched.is_empty() { + println!(" no drift — event-derived state matches chain"); + } else { + for c in &report.mismatched { + println!( + " DRIFT: {} slot {} : {} -> {} (corrected)", + c.address, c.slot, c.old, c.new + ); + } + } + + // 4. A reorg to block 99 purges everything block 100 touched, so the next + // read re-fetches from RPC (the caller re-ingests the canonical logs). + let purge = pipeline.reorg_to(&mut cache, 99); + println!("\n=== reorg to block 99 ==="); + println!( + " purged {} address(es); pool liquidity now re-reads cold/zero: {:?}", + purge.purged.len(), + cache.cached_storage_value(pool, V3_LIQUIDITY_SLOT), + ); + + Ok(()) + } +} diff --git a/tests/event_pipeline.rs b/tests/event_pipeline.rs index 1905f30..d18d224 100644 --- a/tests/event_pipeline.rs +++ b/tests/event_pipeline.rs @@ -609,10 +609,6 @@ mod uniswap_v3 { U256::from(sqrt_price) | (tick24 << 160) | (high << 184) } - #[allow(dead_code)] - fn tick_word(gross: u128, net: i128) -> U256 { - U256::from(gross) | (U256::from(net as u128) << 128) - } fn unpack_tick_word(w: U256) -> (u128, i128) { let gross = u128::try_from(w & U256::from(u128::MAX)).unwrap(); let net = u128::try_from((w >> 128) & U256::from(u128::MAX)).unwrap() as i128; From d7adfd22ad5b29159db43ca5d9882ec4b8e1e596 Mon Sep 17 00:00:00 2001 From: Kai Aldag Date: Tue, 16 Jun 2026 15:33:39 +0100 Subject: [PATCH 19/26] Phase 4: differential ground-truth test for the event processor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validates the event → state pipeline against real EVM execution: run a swap in a ground-truth revm instance and replay ONLY its emitted logs into a twin cache, then assert the token balances and the packed pool slot0 (price/tick) match bit-for-bit. - fixtures/EventGroundTruthPool.sol + test_v3_pool_creation.hex: a faithful UniswapV3-pool stand-in whose slot0 is a Solidity struct with the identical field widths to UniswapV3Pool.Slot0 — so the *compiler* does the real bit-packing and our StateUpdate::SlotMasked is the thing under test. Its swap does real ERC-20 transfers (canonical Transfer logs) + a compiler-masked slot0 update + the canonical Swap event. - tests/event_ground_truth.rs (protocols-gated): deploy two MockERC20 tokens + the pool into a ground-truth cache (deterministic CREATE addresses), seed liquidity, execute a real swap (capture its 2 Transfer + 1 Swap logs), build the identical pre-swap state in a twin cache, feed only the logs through EventPipeline, and assert balances + slot0 + liquidity equal the ground truth. Explicitly checks the slot0 unlocked + observation-index bits survive the masked update. Result: the event-derived state reproduces the ground-truth EVM execution exactly (swapper/pool balances of both tokens, packed slot0, liquidity). Full suite green both feature configs (322 default incl. the new test, 269 --no-default-features); fmt, clippy --all-targets + --lib --no-default-features, doc, bench --no-run. Co-Authored-By: Claude Opus 4.8 (1M context) --- fixtures/EventGroundTruthPool.sol | 126 ++++++++++++ fixtures/README.md | 25 +++ fixtures/test_v3_pool_creation.hex | 1 + tests/event_ground_truth.rs | 317 +++++++++++++++++++++++++++++ 4 files changed, 469 insertions(+) create mode 100644 fixtures/EventGroundTruthPool.sol create mode 100644 fixtures/test_v3_pool_creation.hex create mode 100644 tests/event_ground_truth.rs diff --git a/fixtures/EventGroundTruthPool.sol b/fixtures/EventGroundTruthPool.sol new file mode 100644 index 0000000..510e9e8 --- /dev/null +++ b/fixtures/EventGroundTruthPool.sol @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +pragma solidity ^0.8.20; + +/// @title TestV3Pool +/// @notice A faithful **stand-in** for a UniswapV3 pool used by the event → +/// state differential test (`tests/event_ground_truth.rs`). It is NOT +/// verbatim Uniswap bytecode; instead it reproduces the two things our +/// event decoder actually depends on, and lets the Solidity compiler — +/// not the test author — generate them: +/// 1. The real UniswapV3 `slot0` **storage packing**: `slot0` is a +/// struct with the identical field order/widths as +/// `IUniswapV3PoolState.slot0`, so the compiler packs +/// `sqrtPriceX96` (bits [0,160)), `tick` (int24, [160,184)), +/// `observationIndex`/cardinality/`feeProtocol`/`unlocked` ([184,256)) +/// into one word at storage slot 0 exactly as the real pool does. +/// A swap assigns only `.sqrtPriceX96`/`.tick`, so the compiler emits +/// the masked update that preserves the observation/`unlocked` bits — +/// the exact behavior our `StateUpdate::SlotMasked` must reproduce. +/// 2. The canonical `Swap(...)` event signature, emitted with the same +/// `sqrtPriceX96`/`liquidity`/`tick` values written to storage. +/// +/// @dev Storage layout (mirrors UniswapV3Pool so the slots match +/// `UniswapV3Layout::uniswap`): +/// slot 0: slot0 (packed) +/// slot 1: feeGrowthGlobal0X128 (unused, for layout parity) +/// slot 2: feeGrowthGlobal1X128 (unused) +/// slot 3: protocolFees (unused) +/// slot 4: liquidity (uint128) +/// `token0`/`token1` are immutable (baked into code, not stored), so they do +/// not perturb the slot numbering. +/// +/// The swap *outcome* (amounts, new price/tick/liquidity) is supplied by the +/// caller so the test is deterministic; the pool still performs real ERC-20 +/// transfers (emitting canonical `Transfer` logs from the token contracts) and a +/// real compiler-packed `slot0` update. The price math itself is irrelevant to +/// what the event processor reconstructs — it reads `sqrtPriceX96`/`tick` from the +/// emitted event, never from the pool's internals. +interface IERC20 { + function transfer(address to, uint256 amount) external returns (bool); + function transferFrom(address from, address to, uint256 amount) external returns (bool); +} + +contract TestV3Pool { + /// Identical field order/widths to UniswapV3Pool.Slot0 (one packed word). + struct Slot0 { + uint160 sqrtPriceX96; + int24 tick; + uint16 observationIndex; + uint16 observationCardinality; + uint16 observationCardinalityNext; + uint8 feeProtocol; + bool unlocked; + } + + event Swap( + address indexed sender, + address indexed recipient, + int256 amount0, + int256 amount1, + uint160 sqrtPriceX96, + uint128 liquidity, + int24 tick + ); + + Slot0 public slot0; // slot 0 + uint256 private feeGrowthGlobal0X128; // slot 1 + uint256 private feeGrowthGlobal1X128; // slot 2 + uint256 private protocolFees; // slot 3 + uint128 public liquidity; // slot 4 + + address public immutable token0; + address public immutable token1; + + constructor(address _token0, address _token1) { + token0 = _token0; + token1 = _token1; + } + + /// Set the initial packed `slot0` (with `unlocked = true` and a non-zero + /// observation index, so the differential test can prove those bits survive + /// a swap) and the initial `liquidity`. + function initialize(uint160 sqrtPriceX96, int24 tick, uint16 observationIndex, uint128 _liquidity) + external + { + slot0 = Slot0({ + sqrtPriceX96: sqrtPriceX96, + tick: tick, + observationIndex: observationIndex, + observationCardinality: 1, + observationCardinalityNext: 1, + feeProtocol: 0, + unlocked: true + }); + liquidity = _liquidity; + } + + /// Execute a swap with a caller-specified outcome: pull `amountIn` of the + /// input token (real `transferFrom` → `Transfer` log), send `amountOut` of the + /// output token (real `transfer` → `Transfer` log), update the packed `slot0` + /// price/tick (compiler-masked, preserving the observation/`unlocked` bits) + /// and `liquidity`, then emit the canonical `Swap` event with those values. + function swap( + bool zeroForOne, + uint256 amountIn, + uint256 amountOut, + uint160 newSqrtPriceX96, + int24 newTick, + uint128 newLiquidity + ) external { + address tokenIn = zeroForOne ? token0 : token1; + address tokenOut = zeroForOne ? token1 : token0; + IERC20(tokenIn).transferFrom(msg.sender, address(this), amountIn); + IERC20(tokenOut).transfer(msg.sender, amountOut); + + // Real Uniswap assigns the struct fields individually; the compiler emits + // the masked SSTORE that preserves observation/unlocked. This is exactly + // what `StateUpdate::SlotMasked` must reproduce off the event. + slot0.sqrtPriceX96 = newSqrtPriceX96; + slot0.tick = newTick; + liquidity = newLiquidity; + + int256 amount0 = zeroForOne ? int256(amountIn) : -int256(amountOut); + int256 amount1 = zeroForOne ? -int256(amountOut) : int256(amountIn); + emit Swap(msg.sender, msg.sender, amount0, amount1, newSqrtPriceX96, newLiquidity, newTick); + } +} diff --git a/fixtures/README.md b/fixtures/README.md index ce6f067..b52807f 100644 --- a/fixtures/README.md +++ b/fixtures/README.md @@ -43,3 +43,28 @@ jq -r '.deployedBytecode.object' out/MockERC20.sol/MockERC20.json \ jq -r '.bytecode.object' out/MockERC20.sol/MockERC20.json \ | sed 's/^0x//' > fixtures/mock_erc20_creation.hex ``` + +## `TestV3Pool` + +A faithful UniswapV3-pool **stand-in** (see +[`EventGroundTruthPool.sol`](EventGroundTruthPool.sol)) used by the Phase 4 +differential ground-truth test +([`../tests/event_ground_truth.rs`](../tests/event_ground_truth.rs)). It is *not* +verbatim Uniswap bytecode; it reproduces the two things the event decoder depends +on and lets the compiler generate them: the real `slot0` **struct packing** +(`sqrtPriceX96`/`tick`/observation/`unlocked`, matching `UniswapV3Pool.Slot0`) at +storage slot 0, and the canonical `Swap(...)` event. A `swap` performs real ERC-20 +transfers (canonical `Transfer` logs) and a compiler-masked `slot0` update, so the +test can replay only the emitted logs into a twin cache and assert the +event-derived state matches the ground-truth EVM execution bit-for-bit. + +- `test_v3_pool_creation.hex` — creation bytecode, for `deploy_contract`. The + constructor takes `(address token0, address token1)`; storage mirrors Uniswap + (slot 0 = `slot0`, slot 4 = `liquidity`). + +Regenerate with `solc` (the source is `^0.8.20`-compatible): + +```sh +solc --bin --optimize --optimize-runs 200 --overwrite -o out fixtures/EventGroundTruthPool.sol +cp out/TestV3Pool.bin fixtures/test_v3_pool_creation.hex +``` diff --git a/fixtures/test_v3_pool_creation.hex b/fixtures/test_v3_pool_creation.hex new file mode 100644 index 0000000..9f7230e --- /dev/null +++ b/fixtures/test_v3_pool_creation.hex @@ -0,0 +1 @@ +60c060405234801561000f575f80fd5b5060405161075338038061075383398101604081905261002e91610060565b6001600160a01b039182166080521660a052610091565b80516001600160a01b038116811461005b575f80fd5b919050565b5f8060408385031215610071575f80fd5b61007a83610045565b915061008860208401610045565b90509250929050565b60805160a0516106866100cd5f395f818161026d01528181610297015261030d01525f81816069015281816102bd01526102e701526106865ff3fe608060405234801561000f575f80fd5b5060043610610060575f3560e01c80630dfe1681146100645780631a686502146100a85780631ff1a703146100d35780633850c7bd146101b15780635c02d26614610255578063d21220a714610268575b5f80fd5b61008b7f000000000000000000000000000000000000000000000000000000000000000081565b6040516001600160a01b0390911681526020015b60405180910390f35b6004546100bb906001600160801b031681565b6040516001600160801b03909116815260200161009f565b6101af6100e136600461053b565b6040805160e0810182526001600160a01b0395909516808652600285900b602087015261ffff93909316908501819052600160608601819052608086018190525f60a0870181905260c0909601528454600160c81b6001600160b81b0319909116909317600160a01b62ffffff909516949094029390931763ffffffff60b81b1916600160b81b90930261ffff60c81b1916929092171763ffffffff60d81b1916630100000160d81b17909155600480546001600160801b0319166001600160801b03909216919091179055565b005b5f54610204906001600160a01b03811690600160a01b810460020b9061ffff600160b81b8204811691600160c81b8104821691600160d81b8204169060ff600160e81b8204811691600160f01b90041687565b604080516001600160a01b03909816885260029690960b602088015261ffff94851695870195909552918316606086015291909116608084015260ff1660a0830152151560c082015260e00161009f565b6101af6102633660046105a4565b61028f565b61008b7f000000000000000000000000000000000000000000000000000000000000000081565b5f866102bb577f00000000000000000000000000000000000000000000000000000000000000006102dd565b7f00000000000000000000000000000000000000000000000000000000000000005b90505f8761030b577f000000000000000000000000000000000000000000000000000000000000000061032d565b7f00000000000000000000000000000000000000000000000000000000000000005b6040516323b872dd60e01b8152336004820152306024820152604481018990529091506001600160a01b038316906323b872dd906064016020604051808303815f875af1158015610380573d5f803e3d5ffd5b505050506040513d601f19601f820116820180604052508101906103a49190610608565b5060405163a9059cbb60e01b8152336004820152602481018790526001600160a01b0382169063a9059cbb906044016020604051808303815f875af11580156103ef573d5f803e3d5ffd5b505050506040513d601f19601f820116820180604052508101906104139190610608565b505f80546001600160a01b0387166001600160b81b031990911617600160a01b62ffffff871602178155600480546001600160801b0319166001600160801b0386161790558861046b576104668761062a565b61046d565b875b90505f8961047b5788610484565b6104848861062a565b60408051848152602081018390526001600160a01b038a16818301526001600160801b0388166060820152600289900b60808201529051919250339182917fc42079f94a6350d7e6235f29174924f928cc2ac818eb64fed8004e115fbcca67919081900360a00190a350505050505050505050565b80356001600160a01b038116811461050f575f80fd5b919050565b8035600281900b811461050f575f80fd5b80356001600160801b038116811461050f575f80fd5b5f805f806080858703121561054e575f80fd5b610557856104f9565b935061056560208601610514565b9250604085013561ffff8116811461057b575f80fd5b915061058960608601610525565b905092959194509250565b80151581146105a1575f80fd5b50565b5f805f805f8060c087890312156105b9575f80fd5b86356105c481610594565b955060208701359450604087013593506105e0606088016104f9565b92506105ee60808801610514565b91506105fc60a08801610525565b90509295509295509295565b5f60208284031215610618575f80fd5b815161062381610594565b9392505050565b5f600160ff1b820161064a57634e487b7160e01b5f52601160045260245ffd5b505f039056fea2646970667358221220492fd2050a35f65b8045bdcc2057caf77c1aed86d203b17fd05c21e978a89f0d64736f6c63430008170033 \ No newline at end of file diff --git a/tests/event_ground_truth.rs b/tests/event_ground_truth.rs new file mode 100644 index 0000000..e382d32 --- /dev/null +++ b/tests/event_ground_truth.rs @@ -0,0 +1,317 @@ +//! **Differential ground-truth test** for the event → state pipeline (Phase 4). +//! +//! The decisive correctness check: does feeding the *real emitted logs* of a swap +//! into our event processor reproduce the *exact* state a real EVM execution +//! produced? We run a swap in a ground-truth revm instance and replay only its +//! logs into a twin cache, then assert the token balances and the packed pool +//! `slot0` (price/tick) match bit-for-bit. +//! +//! Setup (an offline stand-in for RPC-fetched state): +//! 1. Deploy two ERC-20 tokens (the `MockERC20` fixture, balances at slot 3) and a +//! `TestV3Pool` (`fixtures/EventGroundTruthPool.sol`) whose `slot0` is a Solidity +//! **struct** with the identical field widths to `UniswapV3Pool.Slot0` — so the +//! *compiler* (not this test) does the real bit-packing, and our +//! `StateUpdate::SlotMasked` is the thing under test. Seed pool liquidity and a +//! swapper balance. +//! 2. Build the identical pre-swap state in a second ("event-driven") cache. The +//! deploy sequence is deterministic, so the token/pool addresses match. +//! 3. Execute a real `swap` against the ground-truth cache (committing) and +//! capture the emitted logs (two ERC-20 `Transfer`s + the canonical `Swap`). +//! 4. Feed only those logs into the event-driven cache via `EventPipeline`, then +//! assert its balances and `slot0` equal the ground-truth cache's. +//! +//! Runs fully offline. Requires the `protocols` feature (the UniswapV3 adapter). +#![cfg(feature = "protocols")] + +mod common; + +use std::sync::Arc; + +use alloy_primitives::aliases::{I24, U160}; +use alloy_primitives::{Address, Bytes, Log, U256, hex, keccak256}; +use alloy_sol_types::{SolCall, SolValue, sol}; +use anyhow::{Result, anyhow}; +use common::{MOCK_ERC20_CREATION_HEX, install_default_account, setup_cache}; +use evm_fork_cache::cache::{EvmCache, V3_LIQUIDITY_SLOT, V3_SLOT0_SLOT}; +use evm_fork_cache::deploy::{build_init_code, encode_constructor_args}; +use evm_fork_cache::events::{DecoderRegistry, EventPipeline}; +use evm_fork_cache::{Erc20TransferDecoder, UniswapV3Decoder, UniswapV3Layout}; +use revm::context::result::ExecutionResult; + +sol! { + interface Token { + function _mint(address to, uint256 amount) external; + function approve(address spender, uint256 amount) external returns (bool); + } + interface Pool { + function initialize(uint160 sqrtPriceX96, int24 tick, uint16 observationIndex, uint128 liquidity) external; + function swap(bool zeroForOne, uint256 amountIn, uint256 amountOut, uint160 newSqrtPriceX96, int24 newTick, uint128 newLiquidity) external; + } +} + +const POOL_CREATION_HEX: &str = include_str!("../fixtures/test_v3_pool_creation.hex"); + +/// The MockERC20 balance mapping slot (`mapping(address => uint256)` at slot 3). +const BALANCE_SLOT: u64 = 3; + +/// Pre-swap parameters, shared by both caches. +const INIT_SQRT_PRICE: u128 = 1u128 << 96; // 2^96 +const INIT_TICK: i32 = 100; +const INIT_OBS_INDEX: u16 = 7; // non-zero, to prove it survives the swap +const INIT_LIQUIDITY: u128 = 1_000_000; +const POOL_RESERVE: u128 = 1_000_000; +const SWAPPER_TOKEN0: u128 = 500_000; + +/// Swap outcome (the test plays the role of the router specifying it). token0 in, +/// token1 out; a *negative* post-swap tick and a full-width `sqrtPriceX96` stress +/// the slot0 packing/sign handling. +const AMOUNT_IN: u128 = 120_000; +const AMOUNT_OUT: u128 = 80_000; +const NEW_TICK: i32 = -50; +const NEW_LIQUIDITY: u128 = 1_050_000; + +fn deployer() -> Address { + Address::repeat_byte(0xd0) +} +fn swapper() -> Address { + Address::repeat_byte(0x5a) +} + +/// Hashed `balanceOf[owner]` storage key. +fn balance_slot(owner: Address) -> U256 { + U256::from_be_bytes(keccak256((owner, U256::from(BALANCE_SLOT)).abi_encode()).0) +} + +fn call(cache: &mut EvmCache, from: Address, to: Address, data: Vec) -> Result<()> { + match cache.call_raw(from, to, Bytes::from(data), true)? { + ExecutionResult::Success { .. } => Ok(()), + other => Err(anyhow!("call to {to} failed: {other:?}")), + } +} + +/// Build the identical pre-swap state in `cache`, returning `(token0, token1, pool)`. +/// +/// The deploy order is fixed, so the deterministic `CREATE` addresses are the same +/// across caches (essential — the captured logs reference these addresses). +fn build_state(cache: &mut EvmCache) -> Result<(Address, Address, Address)> { + install_default_account(cache, Address::ZERO); // coinbase + install_default_account(cache, deployer()); + install_default_account(cache, swapper()); + + // CREATE addresses are deterministic from (deployer, nonce). Pre-install them + // as empty accounts so revm's CREATE collision-check reads the local overlay + // instead of falling through to a (mocked, empty) RPC fetch. + for nonce in 0..3 { + install_default_account(cache, deployer().create(nonce)); + } + + let erc20_creation = hex::decode(MOCK_ERC20_CREATION_HEX.trim())?; + let token0 = cache.deploy_contract( + deployer(), + build_init_code( + &erc20_creation, + // uint8 encodes as a right-aligned 32-byte word, identical to U256. + encode_constructor_args(("Token0".to_string(), "T0".to_string(), U256::from(18))), + ), + )?; + let token1 = cache.deploy_contract( + deployer(), + build_init_code( + &erc20_creation, + encode_constructor_args(("Token1".to_string(), "T1".to_string(), U256::from(18))), + ), + )?; + let pool_creation = hex::decode(POOL_CREATION_HEX.trim())?; + let pool = cache.deploy_contract( + deployer(), + build_init_code(&pool_creation, encode_constructor_args((token0, token1))), + )?; + + // Initialize the pool's packed slot0 + liquidity. + call( + cache, + deployer(), + pool, + Pool::initializeCall { + sqrtPriceX96: U160::from(INIT_SQRT_PRICE), + tick: I24::try_from(INIT_TICK).unwrap(), + observationIndex: INIT_OBS_INDEX, + liquidity: INIT_LIQUIDITY, + } + .abi_encode(), + )?; + + // Seed reserves into the pool and the input balance into the swapper. + for token in [token0, token1] { + call( + cache, + deployer(), + token, + Token::_mintCall { + to: pool, + amount: U256::from(POOL_RESERVE), + } + .abi_encode(), + )?; + } + call( + cache, + deployer(), + token0, + Token::_mintCall { + to: swapper(), + amount: U256::from(SWAPPER_TOKEN0), + } + .abi_encode(), + )?; + // Swapper approves the pool to pull the input. + call( + cache, + swapper(), + token0, + Token::approveCall { + spender: pool, + amount: U256::from(AMOUNT_IN), + } + .abi_encode(), + )?; + + Ok((token0, token1, pool)) +} + +/// Snapshot the four balances + the packed slot0 + liquidity we compare on. +fn observe( + cache: &EvmCache, + t0: Address, + t1: Address, + pool: Address, +) -> Vec<(String, Option)> { + vec![ + ( + "swapper.t0".into(), + cache.cached_storage_value(t0, balance_slot(swapper())), + ), + ( + "swapper.t1".into(), + cache.cached_storage_value(t1, balance_slot(swapper())), + ), + ( + "pool.t0".into(), + cache.cached_storage_value(t0, balance_slot(pool)), + ), + ( + "pool.t1".into(), + cache.cached_storage_value(t1, balance_slot(pool)), + ), + ( + "pool.slot0".into(), + cache.cached_storage_value(pool, V3_SLOT0_SLOT), + ), + ( + "pool.liquidity".into(), + cache.cached_storage_value(pool, V3_LIQUIDITY_SLOT), + ), + ] +} + +#[tokio::test(flavor = "multi_thread")] +async fn event_processor_reproduces_ground_truth_swap() -> Result<()> { + // 1. Ground-truth cache: build state, snapshot the pre-swap state, execute the + // real swap, capture logs + the post-swap state. + let mut truth = setup_cache().await?; + let (token0, token1, pool) = build_state(&mut truth)?; + let pre_swap = observe(&truth, token0, token1, pool); + + let swap_data = Pool::swapCall { + zeroForOne: true, + amountIn: U256::from(AMOUNT_IN), + amountOut: U256::from(AMOUNT_OUT), + newSqrtPriceX96: U160::MAX, // full-width: stresses the [0,160) boundary + newTick: I24::try_from(NEW_TICK).unwrap(), + newLiquidity: NEW_LIQUIDITY, + } + .abi_encode(); + + let logs: Vec = match truth.call_raw(swapper(), pool, Bytes::from(swap_data), true)? { + ExecutionResult::Success { logs, .. } => logs, + other => return Err(anyhow!("ground-truth swap failed: {other:?}")), + }; + // Two ERC-20 Transfers (token0 in, token1 out) + the canonical Swap. + assert_eq!(logs.len(), 3, "expected 2 Transfer logs + 1 Swap log"); + + // 2. Event-driven cache: identical pre-swap state, addresses match. + let mut driven = setup_cache().await?; + let (token0_d, token1_d, pool_d) = build_state(&mut driven)?; + assert_eq!( + (token0, token1, pool), + (token0_d, token1_d, pool_d), + "deterministic deploy addresses must match across caches" + ); + + // Pre-swap, the freshly-built driven cache equals truth's pre-swap snapshot + // (sanity: the deterministic setup really is identical). + assert_eq!(observe(&driven, token0, token1, pool), pre_swap); + + // 3. Feed ONLY the swap's logs into the event pipeline. + let mut registry = DecoderRegistry::new(); + registry.register(Arc::new(Erc20TransferDecoder::new(U256::from( + BALANCE_SLOT, + )))); + registry.register(Arc::new( + UniswapV3Decoder::new().with_pool(pool, UniswapV3Layout::uniswap(60)), + )); + let mut pipeline = EventPipeline::new(registry); + let digest = pipeline.ingest_logs(&mut driven, 1, &logs); + + // All three logs decoded to applied changes; nothing skipped (hot state). + assert_eq!(digest.decoded_logs, 3, "all 3 logs should decode"); + assert!( + !digest.applied.has_skipped(), + "no cold skips: {:?}", + digest.applied.skipped_masks + ); + + // 4. The decisive check: event-driven state == ground-truth state, field by field. + let truth_state = observe(&truth, token0, token1, pool); + let driven_state = observe(&driven, token0, token1, pool); + assert_eq!( + driven_state, truth_state, + "event-driven state must match the ground-truth EVM execution" + ); + + // Spell out the headline invariants explicitly (defensive, human-readable). + let slot0_truth = truth.cached_storage_value(pool, V3_SLOT0_SLOT).unwrap(); + let slot0_driven = driven.cached_storage_value(pool, V3_SLOT0_SLOT).unwrap(); + assert_eq!( + slot0_driven, slot0_truth, + "packed slot0 (price/tick) must match bit-for-bit" + ); + // The price actually moved, and the observation/unlocked bits survived. + assert_ne!(slot0_driven, U256::from(INIT_SQRT_PRICE), "slot0 changed"); + assert_eq!( + (slot0_driven >> 240) & U256::from(1), + U256::from(1), + "unlocked bit preserved" + ); + assert_eq!( + (slot0_driven >> 184) & U256::from(0xFFFF), + U256::from(INIT_OBS_INDEX), + "obs index preserved" + ); + + // Balances match the ground truth (token0 in, token1 out). + assert_eq!( + driven + .cached_storage_value(token0, balance_slot(swapper())) + .unwrap(), + U256::from(SWAPPER_TOKEN0 - AMOUNT_IN), + ); + assert_eq!( + driven + .cached_storage_value(token1, balance_slot(swapper())) + .unwrap(), + U256::from(AMOUNT_OUT), + ); + + Ok(()) +} From b95f5f81ad203402ed478d88920120d181e15c32 Mon Sep 17 00:00:00 2001 From: Kai Aldag Date: Tue, 16 Jun 2026 16:24:41 +0100 Subject: [PATCH 20/26] Phase 5: spec (COW snapshots, Pillar A) + red acceptance contract docs/phase-5-spec.md is the build contract for the memoized-immutable-base snapshot design + overlay buffer/instance reuse, with locked decisions D1-D5. tests/cow_snapshot.rs is the red gate: a differential-equivalence property (create_snapshot must be read-indistinguishable from a retained create_snapshot_deep_clone reference after every mutation kind) plus the overlay reset()/buffer-reuse contract. Red until the implementation lands. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/phase-5-spec.md | 373 +++++++++++++++++++++++++++++++++++++++++ tests/cow_snapshot.rs | 375 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 748 insertions(+) create mode 100644 docs/phase-5-spec.md create mode 100644 tests/cow_snapshot.rs diff --git a/docs/phase-5-spec.md b/docs/phase-5-spec.md new file mode 100644 index 0000000..384ba14 --- /dev/null +++ b/docs/phase-5-spec.md @@ -0,0 +1,373 @@ +# Phase 5 — copy-on-write snapshots (Pillar A) + +> Status: **build contract**. Authored by the overseer before implementation; the +> red acceptance tests in [`../tests/cow_snapshot.rs`](../tests/cow_snapshot.rs) +> and the extended overlay tests pin this contract and gate the deliverable. +> Decisions below are **locked** (resolved with the user) unless marked OPEN. + +## 0. Ground rules + +1. **No behavior change on reads.** Every read against a snapshot or an overlay + built from it must return *exactly* what today's deep-clone snapshot returns — + bit-for-bit, including the audited `StorageCleared` / `NotExisting` / + two-layer-precedence semantics. This is enforced by a **differential-equivalence + test** (§8.1): the new `create_snapshot()` must be read-indistinguishable from + the retained reference `create_snapshot_deep_clone()` after *every* mutation kind. +2. **`Send + Sync` snapshot, `Send` overlay, lock-free reads.** `EvmSnapshot` + stays `Send + Sync`; `EvmOverlay` stays `Send`. Snapshot/overlay reads must not + take a lock and must not regress to a non-`O(1)` lookup (no persistent/HAMT map + on the read path — see Decision D1). The existing `test_snapshot_is_send_sync` + and `test_overlay_is_send` must keep passing unchanged. +3. **No new external dependency.** Structural sharing is achieved with `Arc` over + the per-account storage maps, not a third-party persistent-map crate (D1). +4. **Keep the deep-clone reachable** for A/B benchmarking and as the equivalence + reference (D3). It is retained as `create_snapshot_deep_clone()`. +5. **Standard bars.** `cargo fmt --check`; `cargo clippy --all-targets -- -D + warnings` (default) **and** `cargo clippy --lib --no-default-features -- -D + warnings`; `cargo test` (both feature configs); `RUSTDOCFLAGS=-D warnings cargo + doc`; `cargo bench --no-run`. + +## 1. Goal + +`EvmCache::create_snapshot()` is today an **O(total state) deep clone** +([`mod.rs`](../src/cache/mod.rs) `create_snapshot`): it copies every account and, +dominantly, **every storage slot** of both cache layers into fresh `HashMap`s on +every call. Pillar A replaces this with a **copy-on-write** scheme whose cost +tracks *changed* state, not *total* state, and whose clones are `Arc` handle +copies rather than deep copies. + +Two pillars (both in scope this phase): + +- **A.1 — structural sharing for `create_snapshot`** (§2–§4). +- **A.2 — overlay buffer / instance reuse** (§5). + +## 2. Model: memoized immutable base + fresh hot-layer fold + +### 2.1 The two layers and the cost asymmetry + +- **Layer 2 — `BlockchainDb` (the cold base).** The lazily-fetched / bulk-seeded + fork index. At a fixed block it is **append-mostly**: a fetched `(addr, slot)` + value is canonical and is not rewritten; only `set_block`/re-pin replaces it, + and the controlled bulk writers (`inject_storage_batch*`) and the write-through + funnel mutate it. This is the *large* state. +- **Layer 1 — `CacheDB` overlay (the hot delta).** revm sim commits, write-through + applies, direct inserts, freshness corrections. This is the *small, changing* + set, and it always **shadows** layer 2 on a read (overlay wins). + +The deep clone re-copies all of layer 2 every call even though it barely changes +between successive snapshots. COW memoizes layer 2 and folds only layer 1 fresh. + +### 2.2 The frozen base + +Add an internal, immutable, `Arc`-shared flatten of **layer 2 only**: + +```rust +// src/cache/snapshot.rs (or a new src/cache/cow.rs, implementer's choice) +pub(crate) struct BaseState { + /// Layer-2 account info, by address. (Layer-2 has no NotExisting concept; + /// that classification is purely a layer-1 property — see §4.) + pub(crate) accounts: HashMap, + /// Layer-2 storage, per account, **shared by `Arc`** so cloning a base is a + /// handle copy, never a per-slot copy. + pub(crate) storage: HashMap>>, + /// Bytecode by hash, derived from `accounts` at build time. + pub(crate) code_by_hash: HashMap, +} +``` + +`EvmCache` memoizes the current base and the bookkeeping needed to keep it honest: + +```rust +// fields on EvmCache +base: Option>, // None until first snapshot / after a reset +base_dirty: HashSet
, // layer-2 addrs changed since `base` was built +base_full_rebuild: bool, // set by set_block / re-pin: rebuild from scratch +base_storage_lens: HashMap, // per-acct layer-2 slot counts at last build +``` + +These fields are **not** part of any public API and **not** serialized. + +### 2.3 `refresh_base(&mut self)` — called at the top of `create_snapshot` + +Produces an up-to-date `Arc` reusing the previous one wherever layer 2 +is unchanged. It must **never mutate an `Arc` that may be shared** with a +live snapshot — on any change it builds a *new* `BaseState` that shares the `Arc`s +of unchanged accounts and rebuilds only changed ones (copy-on-write). + +Algorithm: + +1. **Full rebuild** if `base.is_none() || base_full_rebuild`: + flatten all of layer 2 into a fresh `BaseState` (one `Arc` per account); + record `base_storage_lens`; clear `base_dirty`; clear `base_full_rebuild`. +2. **Else, detect uncontrolled growth** (lazy RPC fetch / prefetch writes layer 2 + from inside `foundry-fork-db`, which we cannot hook): scan + `blockchain_db.storage().read()` and `accounts().read()`; for any address whose + slot count differs from `base_storage_lens`, or any account absent from the base, + add it to `base_dirty`. This is an `O(accounts)` length comparison — **not** an + `O(slots)` value scan. +3. **Else, if `base_dirty` is empty** → reuse the existing `Arc` + unchanged (the common hot-loop case; `create_snapshot` is then `O(1)` for the + base). +4. **Otherwise (some addresses dirty)** → build a new `BaseState`: + - clone the outer maps (an `O(accounts)` clone of `Arc` handles + plain + `AccountInfo`, **no per-slot copy**); + - for each dirty address, rebuild its `Arc>` from the current + layer-2 storage and refresh its `AccountInfo` / `code_by_hash`; + - update `base_storage_lens`; clear `base_dirty`; store as the new `Arc`. + +> Correctness rests on `base_dirty` ∪ the growth scan covering every way layer 2 +> can change such that the change is **not shadowed by layer 1**. §3 enumerates the +> sites. The equivalence test (§8.1) exercises all of them and fails loudly on any +> miss — a missed invalidation is a red test, never a silent stale read. + +### 2.4 `create_snapshot()` — the two-tier snapshot + +```rust +pub fn create_snapshot(&mut self) -> Arc { … } +``` + +Note the signature change to `&mut self` (it now refreshes/memoizes the base). +Steps: + +1. `self.refresh_base()` → `let base = Arc::clone(self.base.as_ref().unwrap());` + (`O(1)` when layer 2 is unchanged). +2. Fold **layer 1** (`self.db.cache.accounts`) into the snapshot's overlay maps and + the cleared/not-existing sets, applying the same classification as today + (§4) — `O(layer-1)`. Per-account overlay storage may be a plain + `HashMap` (the hot set is small); `Arc`-interning it is optional. +3. Construct the two-tier `EvmSnapshot { base, …overlay…, …block ctx… }`. + +Block context (`block_number`, `basefee`, `coinbase`, `prevrandao`, `gas_limit`, +`chain_id`, `timestamp`, `spec_id`) is copied as today. + +### 2.5 New `EvmSnapshot` shape + +```rust +pub struct EvmSnapshot { + pub(crate) base: Arc, + /// Layer-1 accounts that are present to the EVM (NotExisting excluded). + pub(crate) overlay_accounts: HashMap, + /// Layer-1 storage delta. A cleared account ALWAYS has an entry here (possibly + /// empty) so the cleared rule is decided without consulting the base. + pub(crate) overlay_storage: HashMap>, + /// Bytecode introduced by layer 1 (checked before `base.code_by_hash`). + pub(crate) overlay_code_by_hash: HashMap, + pub(crate) storage_cleared: HashSet
, + pub(crate) accounts_not_existing: HashSet
, + pub(crate) block_hashes: HashMap, + // …block context fields unchanged… +} +``` + +All fields stay `pub(crate)` (no public field break). In-crate `#[cfg(test)]` +constructors of `EvmSnapshot` (in `overlay.rs`) must be updated to the new shape. + +## 3. Base-invalidation sites (the correctness checklist) + +Every site below must keep the memoized base honest. Implement as a private +helper (e.g. `self.mark_base_dirty(addr)` / `self.invalidate_base()`). + +| Site | Layer touched | Action | +| --- | --- | --- | +| `write_slot_through(addr, …)` | layer 2 always; layer 1 if present | `mark_base_dirty(addr)` (over-invalidation when also in layer 1 is **safe** — it just re-folds that one account; D2 keeps it simple over clever) | +| `inject_storage_batch` / `inject_storage_batch_fresh` | layer 2 only | `mark_base_dirty(addr)` for each touched addr | +| account info / storage seeded into layer 2 (construction, `inject_v2/v3_*` paths that hit layer 2) | layer 2 | `mark_base_dirty(addr)` | +| `purge_*` removing layer-2 entries | layer 2 | `mark_base_dirty(addr)` (or `invalidate_base()` if simpler for account-level purge) | +| `set_block` / `repin_to_block` | replaces layer 2 | `base_full_rebuild = true` | +| revm commit (`call_raw(commit=true)`, session commit) | **layer 1 only** | **nothing** — folded fresh; never makes the base stale | +| direct `db_mut()` inserts (`insert_account_info`/`insert_account_storage`) | **layer 1** | **nothing** — folded fresh | +| uncontrolled lazy RPC fetch / prefetch | layer 2 | caught by the `O(accounts)` growth scan in `refresh_base` step 2 | + +> The litmus test for "needs invalidation": *can this change a layer-2 value that a +> snapshot read would surface (i.e. that layer 1 does not shadow)?* If yes → dirty. +> Layer-1-only writes are always shadowed → never dirty the base. + +## 4. Read semantics (must equal today's flatten, bit-for-bit) + +`EvmSnapshot` exposes the lookups the overlay needs; `EvmOverlay` calls these +instead of indexing fields directly. + +```rust +impl EvmSnapshot { + /// Account info as the EVM sees it. None for NotExisting (do NOT consult base). + pub(crate) fn account_info(&self, a: Address) -> Option<&AccountInfo> { + if self.accounts_not_existing.contains(&a) { return None; } + self.overlay_accounts.get(&a).or_else(|| self.base.accounts.get(&a)) + } + + /// Storage value, mirroring cached_storage_value / today's flatten. + pub fn storage_value(&self, a: Address, s: U256) -> Option { + if let Some(m) = self.overlay_storage.get(&a) { + if let Some(v) = m.get(&s) { return Some(*v); } + if self.storage_cleared.contains(&a) { return Some(U256::ZERO); } // cleared: base dropped + // not cleared: fall through to base + } + if let Some(v) = self.base.storage.get(&a).and_then(|m| m.get(&s)) { + return Some(v); + } + None + } + + pub(crate) fn code(&self, h: B256) -> Option<&Bytecode> { + self.overlay_code_by_hash.get(&h).or_else(|| self.base.code_by_hash.get(&h)) + } +} +``` + +Invariants the equivalence test pins: +- A **cleared** (`StorageCleared`/`NotExisting`) layer-1 account: snapshot holds + only its overlay slots; an absent slot reads `Some(ZERO)`; base slots are never + surfaced (this is why cleared accounts always get an `overlay_storage` entry). +- A **NotExisting** account: `account_info` → `None`, `storage_value` → `Some(ZERO)` + for any slot; excluded from `overlay_accounts`/`overlay_code_by_hash`. +- A **non-cleared** layer-1 account: overlay slot wins, else base slot, else `None`. +- An address only in layer 2: base slot, else `None`. + +`EvmOverlay::{basic, storage, code_by_hash}` are rewritten to: dirty layer → +`snapshot.account_info/storage_value/code` → (the `NotExisting`/`cleared` +short-circuits already live inside those) → `ext_db` fallback (unchanged) → +default. Behavior must match the current overlay exactly (the existing +`overlay.rs` unit tests must keep passing). + +## 5. Overlay buffer / instance reuse (Pillar A.2) + +### 5.1 `EvmOverlay::reset(&mut self)` — recycle one overlay across many sims + +```rust +/// Clear the per-simulation dirty layer so this overlay can be reused for the +/// next simulation against the same snapshot, without reallocating. +pub fn reset(&mut self) { + self.dirty_accounts.clear(); + self.dirty_storage.clear(); + // keep: snapshot Arc, ext_db, the reusable buffer (§5.2) +} +``` + +A worker doing K sims calls `EvmOverlay::new` once and `reset()` between sims +instead of allocating a fresh overlay (+ dirty maps + `Arc` clone) each time. Must +be exactly equivalent to a fresh overlay: a reset overlay reads the pristine +snapshot base again (regression test §8.2). + +### 5.2 Reusable shared-memory buffer (keep `EvmOverlay: Send`) + +Today each `build_evm` / `build_evm_with_inspector` allocates a fresh +`Rc>>` of 64 KB. Reuse it across calls **without** making the +overlay `!Send`: + +- Store the buffer on the overlay as a **plain `Vec`** (`Send`): + `reusable_buffer: Vec` (pre-allocated to `OVERLAY_SHARED_MEMORY_CAPACITY` in + `new`/`reset` keeps it). +- In the **call methods** (`call_raw`, `simulate_with_transfer_tracking`, + `call_raw_with_access_list_with`) that own the full build→transact→revert cycle: + `let buf = std::mem::take(&mut self.reusable_buffer);` **before** the + `with_db(&mut *self)` borrow, move it into a method-local + `Rc::new(RefCell::new(buf))`, build the EVM with that local context, run, then + after the EVM is dropped reclaim `self.reusable_buffer = Rc::try_unwrap(rc).into_inner(); self.reusable_buffer.clear();` + The `Rc` never lives on the overlay → the overlay stays `Send`. +- Refactor the shared body into e.g. `build_evm_with_local(&mut self, local: LocalContext)`; + the **public `build_evm`** keeps allocating a fresh buffer (it hands out the EVM + and cannot reclaim) — documented. +- A panic between take and reclaim only loses the buffer (re-allocated next call); + no correctness impact. + +`test_overlay_is_send` must still compile/pass. + +## 6. Public API surface + +- `EvmCache::create_snapshot(&mut self) -> Arc` — **signature change** + `&self` → `&mut self` (memoizes the base). Update all call sites (freshness + controller, tests, examples, benches). Record in CHANGELOG `### Changed`. +- `EvmCache::create_snapshot_deep_clone(&self) -> Arc` — **new**, + `#[doc(hidden)] pub`. The retained reference: today's flatten producing a + two-tier snapshot with `base` = the fully-merged flatten and empty overlay maps + (plus the cleared/not-existing sets in place). Used by the equivalence test and + the A/B bench. Stays `&self`. +- `EvmOverlay::reset(&mut self)` — **new** public method. +- `EvmSnapshot::storage_value` — retained (reimplemented over two tiers); used by + the freshness validator. `account_info`/`code` are `pub(crate)`. +- No other public signatures change. + +## 7. Benchmarks (`benches/simulation.rs`) + +The current `populated_cache` seeds everything via `db_mut()` into **layer 1**, +which is *not* how a fork cache holds its cold index. Update/extend: + +1. **Realistic cold index in layer 2.** Add a `populated_cache_layer2` that bulk- + seeds the index via `inject_storage_batch` (the cold-load path) so the cold + state lives in the base. The `create_snapshot` group runs the COW path on it. +2. **A/B group.** For each size, bench both `create_snapshot` (COW) and + `create_snapshot_deep_clone` (legacy) so the win is explicit in one report. +3. **Hot-loop re-snapshot.** New bench: build the cold base, take one snapshot + (warms the base), apply a *small* layer-1 mutation (a handful of slots via + `apply_updates`), then measure `create_snapshot` — this is the memoization win + (should be ≈ flat across cold-index size, vs. the deep clone's slope). +4. **`overlay_fanout`.** Add a `reset()`-recycled variant alongside the + `EvmOverlay::new`-per-iter variant to show the A.2 win. + +Keep all benches offline (mocked provider). Document expected shapes in the module +header (COW `create_snapshot` flat vs. deep-clone sloped; re-snapshot ≈ O(changed)). + +## 8. Tests (red contract — written before implementation) + +### 8.1 `tests/cow_snapshot.rs` — differential equivalence (the gate) + +A helper `assert_equivalent(cache)` builds `cow = cache.create_snapshot()` and +`deep = cache.create_snapshot_deep_clone()` and asserts they are +**read-indistinguishable**, comparing via reads (internal reprs differ by design): +- identical account set (union of probed addresses); `basic`/`account_info` equal + for each (including `None` for NotExisting); +- `storage_value(a, s)` equal for every probed `(a, s)` — including **absent** + slots (expect equal `None`/`Some(ZERO)`), cleared accounts, and not-existing + accounts; +- identical `code` for each probed code hash; identical block context; +- overlays built from each (`EvmOverlay::new(snap, None)`) return identical + `balanceOf` / `call_raw` outputs for a `MockERC20`. + +Drive a single cache through a sequence and assert equivalence **after each step**: +1. empty cache; 2. after `insert_account_info`; 3. after layer-1 storage insert; +4. after `apply_updates` `Slot` write-through (addr in layer 1 → shadowed); +5. after `apply_updates` `Slot` write-through to an addr **absent** from layer 1 + (layer-2-only — the §3 footgun); 6. after `apply_updates` `BalanceDelta`; +7. after a committing `call_raw` (revm commit → layer 1); 8. after + `inject_storage_batch` (layer-2-only, incl. **overwriting** an existing slot at + unchanged length); 9. after a simulated lazy fetch (insert directly into + `blockchain_db` to mimic backend growth — both a new account and a **new slot on + an existing account**); 10. after a `purge_*`; 11. after `set_block`. +Also: take a snapshot, then mutate the cache, and assert the **earlier** snapshot is +unchanged (memoized base is COW, not aliased). + +### 8.2 Overlay reuse (extend `tests/snapshot_overlay.rs` or new module) + +- `reset()` clears dirty state: commit a transfer into an overlay, `reset()`, then + reads observe the pristine snapshot again. +- A reset-recycled overlay across two sims yields identical results to two fresh + overlays. +- `EvmOverlay` stays `Send`; `EvmSnapshot` stays `Send + Sync` (compile asserts). +- Buffer reuse does not change call results (a second `call_raw` on the same + overlay returns the same value as the first). + +The existing `tests/snapshot_overlay.rs` cases (immutability, isolation, cleared, +not-existing) must keep passing **unchanged** — they are part of the contract. + +## 9. Locked decisions + +- **D1 — `Arc`-shared maps, not persistent HAMT.** Reads stay `O(1)` with no + per-`SLOAD` regression; no external dependency. (Rejected: `imbl`/`rpds`.) +- **D2 — base memoized as immutable; over-invalidation is acceptable, silent + staleness is not.** `write_slot_through` marks the address dirty unconditionally + (simpler than reasoning per-call about layer-1 shadowing); the equivalence test + is the hard backstop. +- **D3 — keep the deep clone** as `create_snapshot_deep_clone` for A/B + as the + equivalence reference. +- **D4 — overlay reuse: buffer reuse *and* `reset()` recycle** (both in scope). +- **D5 — `create_snapshot` becomes `&mut self`** (the memoization cost). The + freshness controller and all callers are updated. + +## 10. Acceptance + +All §0.5 bars green in both feature configs; the §8 tests pass; the existing +snapshot/overlay/freshness tests pass unchanged; `benches/simulation.rs` shows the +COW `create_snapshot` and the `reset()` fan-out beating their legacy/`new` +baselines, with the numbers reported. CHANGELOG / ROADMAP (Phase 5 → Done) / +KNOWN_ISSUES updated. Lands on `phase-5-cow-snapshots`, stacked on +`phase-4-event-pipeline`. diff --git a/tests/cow_snapshot.rs b/tests/cow_snapshot.rs new file mode 100644 index 0000000..a89007b --- /dev/null +++ b/tests/cow_snapshot.rs @@ -0,0 +1,375 @@ +//! Phase 5 (Pillar A) acceptance tests — the **red contract** for copy-on-write +//! snapshots, authored before implementation. +//! +//! The gate is a *differential-equivalence* property: the new, memoized +//! [`EvmCache::create_snapshot`] must be **read-indistinguishable** from the +//! retained reference [`EvmCache::create_snapshot_deep_clone`] after every kind of +//! cache mutation. Because the two use different internal representations (the COW +//! snapshot shares an `Arc`-ed cold base; the reference is a full flatten), they are +//! compared *through reads only* — `storage_value`, overlay `basic`/`storage`, and a +//! `MockERC20` `balanceOf`. Any base-invalidation miss surfaces here as a failed +//! assertion, never as a silent stale read. +//! +//! Also pins the Pillar A.2 overlay-reuse contract: [`EvmOverlay::reset`] recycles +//! an overlay equivalently to a fresh one, and buffer reuse does not change results. +//! +//! All state is injected over a mocked provider — no test touches the network. +//! +//! These reference `create_snapshot_deep_clone` and `EvmOverlay::reset`, which do +//! not exist until the Phase 5 implementation lands; until then this file fails to +//! compile (red), exactly as intended. + +mod common; + +use std::sync::Arc; + +use alloy_primitives::{Address, U256, keccak256}; +use alloy_sol_types::{SolCall, SolValue}; +use anyhow::{Result, anyhow}; +use revm::database::AccountState; +use revm::database_interface::Database; +use revm::state::AccountInfo; + +use common::{ + MOCK_ERC20_BALANCE_SLOT, MockERC20, install_default_account, install_mock_erc20, setup_cache, + transfer, +}; +use evm_fork_cache::cache::{EvmCache, EvmOverlay, EvmSnapshot}; +use evm_fork_cache::{SlotDelta, StateUpdate}; + +/// `keccak256(abi.encode(owner, slot))` — the hashed mapping slot of +/// `balanceOf[owner]`. +fn mapping_slot(owner: Address, slot: u64) -> U256 { + let key = keccak256((owner, U256::from(slot)).abi_encode()); + U256::from_be_bytes(key.0) +} + +/// Two `AccountInfo`s are equal as the EVM sees them (code identity via code_hash). +fn account_eq(a: &Option, b: &Option) -> bool { + match (a, b) { + (None, None) => true, + (Some(x), Some(y)) => { + x.balance == y.balance + && x.nonce == y.nonce + && x.code_hash == y.code_hash + && x.code.is_some() == y.code.is_some() + } + _ => false, + } +} + +/// Read `balanceOf(owner)` through an overlay (non-committing). +fn overlay_balance_of(overlay: &mut EvmOverlay, token: Address, owner: Address) -> Result { + let call = MockERC20::balanceOfCall { account: owner }; + match overlay.call_raw(owner, token, call.abi_encode().into())? { + revm::context::result::ExecutionResult::Success { output, .. } => Ok( + MockERC20::balanceOfCall::abi_decode_returns(&output.into_data())?, + ), + other => Err(anyhow!("overlay balanceOf failed: {other:?}")), + } +} + +/// Assert the COW snapshot and the deep-clone reference are read-indistinguishable +/// across a probe set of addresses and slots. `label` identifies the mutation step. +fn assert_equivalent(cache: &mut EvmCache, addrs: &[Address], slots: &[U256], label: &str) { + let cow = cache.create_snapshot(); + let deep = cache.create_snapshot_deep_clone(); + + let mut ov_cow = EvmOverlay::new(Arc::clone(&cow), None); + let mut ov_deep = EvmOverlay::new(Arc::clone(&deep), None); + + // Block context must match. + assert_eq!(ov_cow.chain_id(), ov_deep.chain_id(), "{label}: chain_id"); + assert_eq!( + ov_cow.block_number(), + ov_deep.block_number(), + "{label}: block_number" + ); + assert_eq!(ov_cow.basefee(), ov_deep.basefee(), "{label}: basefee"); + assert_eq!(ov_cow.timestamp(), ov_deep.timestamp(), "{label}: timestamp"); + + for &a in addrs { + let bc = ov_cow.basic(a).expect("cow basic"); + let bd = ov_deep.basic(a).expect("deep basic"); + assert!( + account_eq(&bc, &bd), + "{label}: basic mismatch at {a}: cow={bc:?} deep={bd:?}" + ); + for &s in slots { + assert_eq!( + cow.storage_value(a, s), + deep.storage_value(a, s), + "{label}: snapshot.storage_value mismatch at {a} / {s}" + ); + let scow = ov_cow.storage(a, s).expect("cow storage"); + let sdeep = ov_deep.storage(a, s).expect("deep storage"); + assert_eq!( + scow, sdeep, + "{label}: overlay storage mismatch at {a} / {s}" + ); + } + } +} + +/// The core gate: drive one cache through every mutation kind and assert the COW +/// snapshot stays read-identical to the deep-clone reference after each step. +#[tokio::test(flavor = "multi_thread")] +async fn cow_snapshot_matches_deep_clone_through_mutations() -> Result<()> { + let mut cache = setup_cache().await?; + + let token = Address::repeat_byte(0x11); // cleared layer-1 account (MockERC20) + let owner = Address::repeat_byte(0x22); + let recipient = Address::repeat_byte(0x33); + let pool = Address::repeat_byte(0x77); // layer-2-only, non-cleared + let pool2 = Address::repeat_byte(0x88); // write-through target absent from layer 1 + let pool3 = Address::repeat_byte(0x99); // appears via simulated lazy fetch + let ghost = Address::repeat_byte(0xEE); // becomes NotExisting + + let balance_slot = U256::from(MOCK_ERC20_BALANCE_SLOT); + let owner_bal = mapping_slot(owner, MOCK_ERC20_BALANCE_SLOT); + let recip_bal = mapping_slot(recipient, MOCK_ERC20_BALANCE_SLOT); + + let addrs = [token, owner, recipient, pool, pool2, pool3, ghost]; + // Probe real slots plus an always-absent slot (both must agree on None). + let slots = [ + balance_slot, + owner_bal, + recip_bal, + U256::from(0u64), + U256::from(1u64), + U256::from(7u64), + U256::from(424_242u64), // never set anywhere + ]; + + // 1. Empty cache. + assert_equivalent(&mut cache, &addrs, &slots, "empty"); + + // 2. Layer-1 account inserts. + install_default_account(&mut cache, Address::ZERO); + install_default_account(&mut cache, owner); + install_default_account(&mut cache, recipient); + install_mock_erc20(&mut cache, token); + assert_equivalent(&mut cache, &addrs, &slots, "after account inserts"); + + // 3. Layer-1 storage inserts (mapping balances on the cleared token). + cache.insert_mapping_storage_slot(token, balance_slot, owner, U256::from(1_000u64))?; + cache.insert_mapping_storage_slot(token, balance_slot, recipient, U256::ZERO)?; + assert_equivalent(&mut cache, &addrs, &slots, "after layer-1 storage"); + + // 4. write-through to an address PRESENT in layer 1 (shadowed there). + cache.apply_updates(&[StateUpdate::slot(token, owner_bal, U256::from(2_000u64))]); + assert_equivalent(&mut cache, &addrs, &slots, "after write-through (in layer 1)"); + + // 5. write-through to an address ABSENT from layer 1 (layer-2-only — the §3 + // footgun: the base must capture it). + cache.apply_updates(&[StateUpdate::slot(pool2, U256::from(7u64), U256::from(55u64))]); + assert_equivalent(&mut cache, &addrs, &slots, "after write-through (layer-2-only)"); + + // 6. relative native-balance delta. + cache.apply_updates(&[StateUpdate::balance_delta(owner, SlotDelta::Add(U256::from(500)))]); + assert_equivalent(&mut cache, &addrs, &slots, "after balance delta"); + + // 7. committing revm call (mutates layer 1 only — never stales the base). + transfer(&mut cache, token, owner, recipient, U256::from(250u64))?; + assert_equivalent(&mut cache, &addrs, &slots, "after committed transfer"); + + // 8. layer-2-only cold backfill, including OVERWRITING an existing slot at an + // unchanged length (must still invalidate the base). + cache.inject_storage_batch(&[(pool, U256::from(0u64), U256::from(111u64))]); + assert_equivalent(&mut cache, &addrs, &slots, "after inject (new)"); + cache.inject_storage_batch(&[(pool, U256::from(0u64), U256::from(222u64))]); + assert_equivalent(&mut cache, &addrs, &slots, "after inject (overwrite, same len)"); + + // 9. simulated UNCONTROLLED layer-2 growth (a lazy RPC fetch / prefetch writes + // `BlockchainDb` from inside foundry-fork-db, bypassing our write funnel): + // a brand-new account+slot, and a NEW slot on the existing `pool`. + { + let bdb = cache.blockchain_db(); + bdb.storage() + .write() + .entry(pool3) + .or_default() + .insert(U256::from(1u64), U256::from(909u64)); + bdb.accounts().write().insert( + pool3, + AccountInfo { + balance: U256::from(5u64), + ..Default::default() + }, + ); + // New slot on an existing base account (len changes → growth scan must catch). + bdb.storage() + .write() + .entry(pool) + .or_default() + .insert(U256::from(1u64), U256::from(333u64)); + } + assert_equivalent(&mut cache, &addrs, &slots, "after uncontrolled layer-2 growth"); + + // 10. purge. + cache.purge_account(owner); + assert_equivalent(&mut cache, &addrs, &slots, "after purge_account"); + + // 11. NotExisting account (absent to the EVM; storage reads ZERO). + cache.db_mut().insert_account_info( + ghost, + AccountInfo { + balance: U256::from(1u64), + ..Default::default() + }, + ); + cache + .db_mut() + .cache + .accounts + .get_mut(&ghost) + .expect("ghost present") + .account_state = AccountState::NotExisting; + assert_equivalent(&mut cache, &addrs, &slots, "after NotExisting"); + + // 12. set_block (re-pin → full base rebuild path). + cache.set_block(None); + assert_equivalent(&mut cache, &addrs, &slots, "after set_block"); + + Ok(()) +} + +/// COW must not alias: a snapshot taken earlier is unaffected by a later mutation +/// of the same address (the memoized base is rebuilt copy-on-write, not mutated). +#[tokio::test(flavor = "multi_thread")] +async fn earlier_snapshot_unaffected_by_later_base_mutation() -> Result<()> { + let mut cache = setup_cache().await?; + let pool = Address::repeat_byte(0x77); + let slot = U256::from(3u64); + + cache.inject_storage_batch(&[(pool, slot, U256::from(100u64))]); + let early = cache.create_snapshot(); + assert_eq!(early.storage_value(pool, slot), Some(U256::from(100u64))); + + // Mutate the same base slot, then take a second snapshot. + cache.inject_storage_batch(&[(pool, slot, U256::from(200u64))]); + let late = cache.create_snapshot(); + + assert_eq!( + early.storage_value(pool, slot), + Some(U256::from(100u64)), + "the earlier snapshot must still read the pre-mutation value" + ); + assert_eq!( + late.storage_value(pool, slot), + Some(U256::from(200u64)), + "the later snapshot reflects the mutation" + ); + Ok(()) +} + +/// `EvmOverlay::reset` clears the dirty layer so the overlay reads the pristine +/// snapshot again — equivalent to a fresh overlay. +#[tokio::test(flavor = "multi_thread")] +async fn overlay_reset_restores_pristine_snapshot_reads() -> Result<()> { + let mut cache = setup_cache().await?; + let token = Address::repeat_byte(0x44); + let owner = Address::repeat_byte(0x55); + let recipient = Address::repeat_byte(0x66); + + install_default_account(&mut cache, Address::ZERO); + install_default_account(&mut cache, owner); + install_default_account(&mut cache, recipient); + install_mock_erc20(&mut cache, token); + cache.insert_mapping_storage_slot( + token, + U256::from(MOCK_ERC20_BALANCE_SLOT), + owner, + U256::from(10_000u64), + )?; + cache.insert_mapping_storage_slot( + token, + U256::from(MOCK_ERC20_BALANCE_SLOT), + recipient, + U256::ZERO, + )?; + + let snapshot = cache.create_snapshot(); + let mut overlay = EvmOverlay::new(Arc::clone(&snapshot), None); + + // Mutate the dirty layer with a committing transfer through the overlay. + overlay.simulate_with_transfer_tracking( + owner, + token, + MockERC20::transferCall { + to: recipient, + amount: U256::from(4_000u64), + } + .abi_encode() + .into(), + owner, + Some([token]), + true, // commit into the overlay's dirty layer + )?; + assert_eq!( + overlay_balance_of(&mut overlay, token, owner)?, + U256::from(6_000u64), + "post-transfer dirty-layer balance" + ); + + // reset() drops the dirty layer; reads see the pristine snapshot again. + overlay.reset(); + assert_eq!( + overlay_balance_of(&mut overlay, token, owner)?, + U256::from(10_000u64), + "after reset the overlay reads the pristine snapshot" + ); + + // A reset-recycled overlay matches a brand-new overlay across two sims. + let mut fresh = EvmOverlay::new(Arc::clone(&snapshot), None); + assert_eq!( + overlay_balance_of(&mut overlay, token, owner)?, + overlay_balance_of(&mut fresh, token, owner)?, + "recycled overlay == fresh overlay" + ); + Ok(()) +} + +/// Buffer reuse must not change results: repeated calls on one overlay return the +/// same value as the first (the reusable shared-memory buffer is cleared, not +/// corrupted, between builds). +#[tokio::test(flavor = "multi_thread")] +async fn overlay_buffer_reuse_is_result_stable() -> Result<()> { + let mut cache = setup_cache().await?; + let token = Address::repeat_byte(0x44); + let owner = Address::repeat_byte(0x55); + + install_default_account(&mut cache, Address::ZERO); + install_default_account(&mut cache, owner); + install_mock_erc20(&mut cache, token); + cache.insert_mapping_storage_slot( + token, + U256::from(MOCK_ERC20_BALANCE_SLOT), + owner, + U256::from(7_777u64), + )?; + + let snapshot = cache.create_snapshot(); + let mut overlay = EvmOverlay::new(snapshot, None); + + let first = overlay_balance_of(&mut overlay, token, owner)?; + for _ in 0..16 { + assert_eq!( + overlay_balance_of(&mut overlay, token, owner)?, + first, + "repeated calls reusing the buffer must be stable" + ); + } + assert_eq!(first, U256::from(7_777u64)); + Ok(()) +} + +/// Compile-time guards: the COW representation keeps the thread-safety contract. +#[test] +fn snapshot_send_sync_overlay_send() { + fn assert_send_sync() {} + fn assert_send() {} + assert_send_sync::(); + assert_send_sync::>(); + assert_send::(); +} From 3c407c66b18f0e01fffc16bf0b3ecc58d4377451 Mon Sep 17 00:00:00 2001 From: Kai Aldag Date: Tue, 16 Jun 2026 17:10:52 +0100 Subject: [PATCH 21/26] Phase 5: copy-on-write snapshots (Pillar A) + overlay reuse Replace the O(total state) deep-clone create_snapshot with a two-tier copy-on-write snapshot: the cold layer-2 BlockchainDb index is flattened once into an immutable Arc (per-account storage shared by Arc), memoized across snapshots and rebuilt copy-on-write only for changed addresses; each snapshot folds just the hot layer-1 CacheDB delta over a cheap Arc::clone. Reads stay O(1) and lock-free (no persistent-map dep, D1); EvmSnapshot stays Send+Sync. create_snapshot is now &mut self (D5); create_snapshot_deep_clone is retained as the A/B baseline and the differential read-equivalence reference (D3). Every controlled layer-2 write marks the base dirty; an O(accounts) length-scan catches the append-only lazy-fetch growth. Overlay reuse (D4): EvmOverlay::reset() recycles an overlay across sims, and the 64KB shared-memory buffer is reused across calls via a Send-preserving take/reclaim (plain Vec field, method-local Rc) instead of re-allocating per build. Indicative: create_snapshot ~30-60x faster than the deep clone on the cold-index sweep; reset()-recycled fan-out beats fresh-overlay. Overseer review + adversarial-panel remediation: - mark_base_dirty in override_account_code_with_missing_target (D2 uniformity). - invalidate_snapshot_base() public re-honest hook + rustdoc warnings on blockchain_db()/backend() + a load-bearing-invariant note in refresh_base, for the one residual edge (an out-of-band same-length layer-2 overwrite through the public escape hatches); pinned by a guard test and recorded in KNOWN_ISSUES. Tests: 328 (default) / 275 (--no-default-features), incl. the cow_snapshot differential gate (COW == deep-clone after every mutation kind) and overlay-reuse contract. fmt + clippy (both configs) + doc + bench --no-run clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 29 +++ benches/simulation.rs | 143 ++++++++---- benches/state_update.rs | 8 +- docs/KNOWN_ISSUES.md | 38 +++- docs/ROADMAP.md | 52 ++++- src/cache/mod.rs | 463 ++++++++++++++++++++++++++++++++++---- src/cache/overlay.rs | 488 +++++++++++++++++++++++----------------- src/cache/snapshot.rs | 160 +++++++++---- tests/cow_snapshot.rs | 82 ++++++- 9 files changed, 1119 insertions(+), 344 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0968cf3..11dcc9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -142,9 +142,38 @@ pre-release development phases (see [`docs/ROADMAP.md`](docs/ROADMAP.md)). - **`protocols` feature** (default-on) gating the Uniswap V2/V3 storage layouts, V3 tick snapshots, and `inject_v3_*` / `inject_v2_pool_metadata` helpers, so the generic engine builds with `--no-default-features`. +- **Copy-on-write snapshots** (Phase 5, Pillar A) — `create_snapshot` is now a + two-tier copy-on-write view instead of an O(total state) deep clone. The cold + `BlockchainDb` index (layer 2) is flattened once into an internal, immutable, + `Arc`-shared base (`Arc` per account storage map, structural sharing — no new + dependency, Decision D1), memoized across snapshots and rebuilt copy-on-write + only for the addresses that changed; each `create_snapshot` then folds just the + hot CacheDB delta (layer 1) over a cheap `Arc::clone` of that base. Reads stay + O(1) and lock-free and are bit-for-bit identical to the deep clone (pinned by + the `tests/cow_snapshot.rs` differential-equivalence gate). The retained + `EvmCache::create_snapshot_deep_clone()` (`#[doc(hidden)] pub`, Decision D3) is + the equivalence reference and the A/B benchmark baseline. `EvmSnapshot` stays + `Send + Sync` and `EvmOverlay` stays `Send`. +- **`EvmOverlay::reset()`** (Phase 5, Pillar A.2) — recycle one overlay across + many simulations against the same snapshot without reallocating: it clears the + per-simulation dirty layer (keeping the snapshot `Arc`, `ext_db`, and the + reusable shared-memory buffer), reading the pristine snapshot again and behaving + exactly like a freshly-built overlay. The 64 KB shared-memory buffer is also + recycled across the build→transact→revert call methods (stored as a plain + `Vec`, so the overlay stays `Send`). ### Changed +- **`EvmCache::create_snapshot` is now `&mut self`** (Phase 5, Decision D5) — + taking a snapshot memoizes/refreshes the cold copy-on-write base, which requires + a mutable borrow. All callers (the freshness controller, tests, examples, + benches) are updated; the return type (`Arc`) is unchanged. + Permitted under the pre-1.0 break policy. +- **`EvmCache::inject_storage_batch` is now `&mut self`** (Phase 5) — the + layer-2 bulk write now marks the touched addresses dirty for the memoized + copy-on-write base. The write itself is still a direct backend (layer-2) write + with the same semantics; only the receiver mutability changed. + - Simulation entry points that distinguish failure modes return `SimulationResult` (`Result`), separating decoded reverts, EVM halts, and host errors. `SimulationErrorKind` remains as a deprecated alias. diff --git a/benches/simulation.rs b/benches/simulation.rs index 10d0d2d..82a8bd2 100644 --- a/benches/simulation.rs +++ b/benches/simulation.rs @@ -3,14 +3,23 @@ //! bundle simulation, and batched storage injection. //! //! These run fully offline (mocked provider) so they're reproducible. They -//! establish the baseline for the Pillar A (copy-on-write snapshot) rewrite: -//! `create_snapshot` is currently an O(total state) deep clone, so its cost -//! scales with the populated cache size (the `create_snapshot` group sweeps -//! 100 → 10,000 accounts to show that slope). Once Pillar A lands, the same -//! sweep should flatten toward O(changed state) — re-run this group before and -//! after to quantify the win. The `overlay_fanout` group measures the other -//! half of the value proposition: how cheaply one frozen snapshot fans out into -//! many isolated simulations. +//! quantify the Pillar A (copy-on-write snapshot) win. +//! +//! Expected shapes: +//! - **`create_snapshot` group (A/B).** The cold index is seeded into **layer 2** +//! via `inject_storage_batch` (`populated_cache_layer2`), the way a fork cache +//! actually holds it. For each size it benches both the COW `create_snapshot` +//! and the retained `create_snapshot_deep_clone`. The deep clone is an O(total +//! state) copy, so its cost slopes up with the index size; the COW path folds +//! only the (empty) hot layer over an `Arc`-shared memoized base, so after the +//! base is warm it should stay roughly **flat** across sizes. +//! - **`resnapshot_hot_loop`.** Warms the base with one snapshot, applies a small +//! `apply_updates` layer-1 mutation, then measures `create_snapshot`. This is +//! the memoization win: ≈ O(changed) and flat across cold-index size, vs. the +//! deep clone's slope. +//! - **`overlay_fanout`.** Measures fanning one frozen snapshot out into many +//! isolated simulations, comparing a fresh `EvmOverlay::new` per sim against a +//! single `reset()`-recycled overlay (Pillar A.2). use std::hint::black_box; use std::sync::Arc; @@ -49,34 +58,35 @@ fn offline_cache(rt: &Runtime) -> EvmCache { rt.block_on(EvmCache::new(Arc::new(provider), None)) } -/// A cache populated with `accounts` accounts, each holding `slots_per` slots. -fn populated_cache(rt: &Runtime, accounts: usize, slots_per: usize) -> EvmCache { +/// A cache whose cold index lives in **layer 2** — seeded via +/// `inject_storage_batch`, the path a fork cache actually uses to bulk-load its +/// cold state. This is what the COW `create_snapshot` memoizes into its base. +fn populated_cache_layer2(rt: &Runtime, accounts: usize, slots_per: usize) -> EvmCache { let mut cache = offline_cache(rt); + let mut batch: Vec<(Address, U256, U256)> = Vec::with_capacity(accounts * slots_per); for a in 0..accounts { let address = addr(a); - cache - .db_mut() - .insert_account_info(address, AccountInfo::default()); for s in 0..slots_per { - cache - .db_mut() - .insert_account_storage( - address, - U256::from(s as u64), - U256::from((a * 31 + s) as u64), - ) - .unwrap(); + batch.push(( + address, + U256::from(s as u64), + U256::from((a * 31 + s) as u64), + )); } } + cache.inject_storage_batch(&batch); cache } +/// A/B snapshot creation across cold-index sizes: the COW `create_snapshot` vs. +/// the retained `create_snapshot_deep_clone`, both over a layer-2-seeded index. +/// +/// The deep clone slopes up with the index; the COW path, after a warm-up +/// snapshot has memoized the base, should stay roughly flat (the hot layer is +/// empty, so it is an `Arc` handle copy plus the O(accounts) growth scan). fn bench_create_snapshot(c: &mut Criterion) { let rt = Runtime::new().unwrap(); let mut group = c.benchmark_group("create_snapshot"); - // Sweep from a small pool up to a production-scale index (10k contracts) so - // the O(total state) slope of the current deep clone is visible. Pillar A - // (copy-on-write) should flatten this curve. for &(accounts, slots) in &[ (100usize, 8usize), (1_000, 8), @@ -84,12 +94,49 @@ fn bench_create_snapshot(c: &mut Criterion) { (5_000, 16), (10_000, 16), ] { - let cache = populated_cache(&rt, accounts, slots); + let mut cache = populated_cache_layer2(&rt, accounts, slots); + // Warm the memoized base once so the COW measurement reflects the + // steady-state (reuse) cost, not the first full build. + black_box(cache.create_snapshot()); + group.throughput(criterion::Throughput::Elements((accounts * slots) as u64)); + group.bench_with_input( + BenchmarkId::new("cow", format!("{accounts}acct_x{slots}slot")), + &accounts, + |b, _| b.iter(|| black_box(cache.create_snapshot())), + ); + group.bench_with_input( + BenchmarkId::new("deep_clone", format!("{accounts}acct_x{slots}slot")), + &accounts, + |b, _| b.iter(|| black_box(cache.create_snapshot_deep_clone())), + ); + } + group.finish(); +} + +/// The memoization win: a hot re-snapshot loop. Warm the base once, apply a +/// *small* layer-1 mutation, then measure `create_snapshot`. Cost should track +/// the changed state (≈ flat across cold-index size), unlike the deep clone. +fn bench_resnapshot_hot_loop(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + let mut group = c.benchmark_group("resnapshot_hot_loop"); + for &(accounts, slots) in &[(1_000usize, 8usize), (5_000, 16), (10_000, 16)] { + let mut cache = populated_cache_layer2(&rt, accounts, slots); + // Warm the base. + black_box(cache.create_snapshot()); + // A handful of layer-1 writes (does not dirty the memoized base). + let target = addr(0); group.throughput(criterion::Throughput::Elements((accounts * slots) as u64)); group.bench_with_input( BenchmarkId::from_parameter(format!("{accounts}acct_x{slots}slot")), - &cache, - |b, cache| b.iter(|| black_box(cache.create_snapshot())), + &accounts, + |b, _| { + b.iter(|| { + cache + .db_mut() + .insert_account_info(target, AccountInfo::default()); + black_box(cache.create_snapshot()); + }) + }, ); } group.finish(); @@ -125,20 +172,31 @@ fn bench_overlay_fanout(c: &mut Criterion) { let mut group = c.benchmark_group("overlay_fanout"); for &k in &[1usize, 8, 32] { - group.bench_with_input( - BenchmarkId::from_parameter(format!("{k}way")), - &k, - |b, &k| { - b.iter(|| { - for _ in 0..k { - let mut overlay = EvmOverlay::new(snapshot.clone(), None); - let result = overlay.call_raw(owner, token, calldata.clone()).unwrap(); - debug_assert!(matches!(result, ExecutionResult::Success { .. })); - black_box(result); - } - }) - }, - ); + // Baseline: a fresh `EvmOverlay::new` (+ dirty maps + Arc clone + buffer) + // per simulation. + group.bench_with_input(BenchmarkId::new("new_per_sim", k), &k, |b, &k| { + b.iter(|| { + for _ in 0..k { + let mut overlay = EvmOverlay::new(snapshot.clone(), None); + let result = overlay.call_raw(owner, token, calldata.clone()).unwrap(); + debug_assert!(matches!(result, ExecutionResult::Success { .. })); + black_box(result); + } + }) + }); + // Pillar A.2: one overlay built once, `reset()` between sims (reuses the + // dirty maps, the snapshot Arc, and the shared-memory buffer). + group.bench_with_input(BenchmarkId::new("reset_recycled", k), &k, |b, &k| { + b.iter(|| { + let mut overlay = EvmOverlay::new(snapshot.clone(), None); + for _ in 0..k { + let result = overlay.call_raw(owner, token, calldata.clone()).unwrap(); + debug_assert!(matches!(result, ExecutionResult::Success { .. })); + black_box(result); + overlay.reset(); + } + }) + }); } group.finish(); } @@ -253,7 +311,7 @@ fn bench_sim_bundle(c: &mut Criterion) { /// Batched direct storage injection (the bypass-RPC write path) across sizes. fn bench_inject_storage_batch(c: &mut Criterion) { let rt = Runtime::new().unwrap(); - let cache = offline_cache(&rt); + let mut cache = offline_cache(&rt); let mut group = c.benchmark_group("inject_storage_batch"); for &n in &[100usize, 1_000, 10_000] { @@ -271,6 +329,7 @@ fn bench_inject_storage_batch(c: &mut Criterion) { criterion_group!( benches, bench_create_snapshot, + bench_resnapshot_hot_loop, bench_overlay_fanout, bench_cache_call_raw, bench_sim_bundle, diff --git a/benches/state_update.rs b/benches/state_update.rs index 907954c..8bf74e3 100644 --- a/benches/state_update.rs +++ b/benches/state_update.rs @@ -181,7 +181,7 @@ fn bench_apply_per_variant(c: &mut Criterion) { group.bench_function("purge_all_storage", |b| { b.iter_batched( || { - let cache = pool_cache(&rt); + let mut cache = pool_cache(&rt); cache.inject_storage_batch(&[ (POOL, U256::from(0), U256::from(1)), (POOL, U256::from(1), U256::from(2)), @@ -203,7 +203,7 @@ fn bench_apply_per_variant(c: &mut Criterion) { group.bench_function("purge_account", |b| { b.iter_batched( || { - let cache = pool_cache(&rt); + let mut cache = pool_cache(&rt); cache.inject_storage_batch(&[ (POOL, U256::from(0), U256::from(1)), (POOL, U256::from(1), U256::from(2)), @@ -223,7 +223,7 @@ fn bench_apply_per_variant(c: &mut Criterion) { group.bench_function("purge_slots", |b| { b.iter_batched( || { - let cache = pool_cache(&rt); + let mut cache = pool_cache(&rt); cache.inject_storage_batch(&[ (POOL, U256::from(0), U256::from(1)), (POOL, U256::from(1), U256::from(2)), @@ -254,7 +254,7 @@ fn bench_apply_heterogeneous(c: &mut Criterion) { group.bench_function("slot_account_purge", |b| { b.iter_batched( || { - let cache = pool_cache(&rt); + let mut cache = pool_cache(&rt); cache.inject_storage_batch(&[(POOL, U256::from(9), U256::from(1))]); cache }, diff --git a/docs/KNOWN_ISSUES.md b/docs/KNOWN_ISSUES.md index ce24b7c..329c2a8 100644 --- a/docs/KNOWN_ISSUES.md +++ b/docs/KNOWN_ISSUES.md @@ -131,9 +131,41 @@ Confidence legend: **[V]** verified against the source during review; ## Limitations by design / roadmap -- **No copy-on-write snapshots yet.** `create_snapshot()` deep-clones state - (`O(accounts + slots)`); the COW rewrite is roadmap Pillar A. The `simulation` - benchmarks exist to measure the baseline this will improve on. +- **Copy-on-write snapshots (Phase 5, Pillar A) — done.** `create_snapshot()` is + no longer an O(total state) deep clone. The cold `BlockchainDb` index (layer 2) + is flattened once into an internal, immutable, `Arc`-shared base (per-account + storage shared by `Arc`), memoized across snapshots and rebuilt copy-on-write + only for the addresses that changed; each snapshot folds just the hot CacheDB + delta (layer 1) over a cheap `Arc::clone`. **Residual cost model (honest):** a + snapshot is no longer free. When layer 2 is unchanged since the last snapshot + it still pays an **O(accounts) length-scan** of the layer-2 storage/account + maps (to catch uncontrolled lazy-fetch growth that bypasses the write funnel, + since `foundry-fork-db` cannot be hooked) plus an **O(layer-1) fold** of the hot + delta — so the cost tracks `accounts + changed state`, not total slots. A + full rebuild (first snapshot, or after `set_block`/re-pin) is still O(total + state). `create_snapshot` is now `&mut self` (it memoizes the base, Decision + D5). The retained `create_snapshot_deep_clone()` (the legacy full flatten) is + kept as the A/B benchmark baseline and the read-equivalence reference; the + `create_snapshot` group in `benches/simulation.rs` measures both. Decisions and + the cost model are in [`phase-5-spec.md`](phase-5-spec.md) / `ROADMAP.md`. +- **[V] Memoized-base staleness at the layer-2 escape hatches (Phase 5).** The + snapshot base's growth scan is count/absence-based, which is sufficient for the + supported writers: the crate's own mutators (`apply_update`, `inject_storage_batch`, + the `inject_*` helpers, purges, code overrides) explicitly mark the base dirty, + and the `foundry-fork-db` `SharedBackend` lazy fetch is append-only at a fixed + block (it only inserts on a cache miss, never overwrites in place — a load-bearing + invariant noted in `refresh_base`). The one gap, surfaced by the Phase 5 + adversarial review: a **direct, out-of-band write through the public + `blockchain_db()` / `backend()` handles** that *overwrites an existing slot value + at an unchanged slot count* is invisible to the scan, so a subsequent + `create_snapshot` may reuse a stale base (`create_snapshot_deep_clone` always + re-reads and would diverge). This is a contract boundary, not an internal bug — no + in-crate path triggers it, and both accessors are documented as bypassing the + two-layer model. Mitigation: call the new + [`EvmCache::invalidate_snapshot_base`] after any direct layer-2 write through those + handles (or re-pin via `set_block`); the rustdoc on both accessors and the hook + carries this warning, and `tests/cow_snapshot.rs` + (`invalidate_snapshot_base_rehonest_after_escape_hatch_write`) pins it. - **`protocols` not yet extracted.** The DeFi surface is feature-gated but still in-crate; `cargo test --no-default-features` is not yet supported because some unit tests assume the default feature. Extraction into `evm-amm-state` is diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 8eaa262..36d2fef 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -74,7 +74,7 @@ RPC node Event-driven sync ← WS logs · new block | **2** | Freshness core (Pillar C): `Validity` + `FreshnessRegistry`; observation tracker; policies; optimistic verify-and-rerun loop. | **Done** (`phase-2-freshness`) | | **3** | State-update primitives (Pillar B.1): `StateUpdate` + targeted writers; refold `inject_*`; surface state-diff output. | **Done** (`phase-3-state-updates`) | | **4** | Event pipeline + adapters (Pillar B.2): `EventDecoder` trait, ERC-20 + V3 adapters, ingest/reorg/reconcile pipeline. | **Done** (`phase-4-event-pipeline`) | -| **5** | COW snapshots (Pillar A): structural sharing; overlay buffer reuse. | Planned | +| **5** | COW snapshots (Pillar A): structural sharing; overlay buffer reuse. | **Done** (`phase-5-cow-snapshots`) | Cross-cutting (land opportunistically): call tracer Inspector, full offline (`default-features = false`, no provider) build split, CHANGELOG/CONTRIBUTING. @@ -406,6 +406,56 @@ is recorded in `KNOWN_ISSUES.md`. --- +## Phase 5 — copy-on-write snapshots (detailed, decisions locked) + +Builds **Pillar A**: replace the O(total state) deep-clone `create_snapshot` with +a two-tier copy-on-write view whose cost tracks *changed* state, not *total* +state. The cold `BlockchainDb` index (layer 2) is flattened once into an +internal, immutable, `Arc`-shared base — both the base as a whole and each +account's storage map are shared by `Arc`, so structural sharing needs no new +dependency (Decision D1) — memoized across snapshots and rebuilt copy-on-write +only for the addresses that changed; each snapshot then folds just the hot +CacheDB delta (layer 1). Reads stay O(1), lock-free, and bit-for-bit identical to +the deep clone. The full build contract is in +[`phase-5-spec.md`](phase-5-spec.md). + +### Locked decisions + +1. **`Arc`-shared maps, not a persistent HAMT** (D1). Reads stay O(1) with no + per-`SLOAD` regression and no external dependency. +2. **Base memoized as immutable; over-invalidation is acceptable, silent + staleness is not** (D2). The write-through funnel marks an address dirty + unconditionally; the differential-equivalence test is the hard backstop. +3. **Keep the deep clone** as `create_snapshot_deep_clone` (D3) — the A/B + benchmark baseline and the read-equivalence reference. +4. **Overlay reuse: buffer reuse *and* `reset()` recycle** (D4) — both in scope. +5. **`create_snapshot` becomes `&mut self`** (D5) — the memoization cost; the + freshness controller and all callers are updated. + +### Acceptance — met + +`cargo fmt --check`, `clippy --all-targets -- -D warnings` (default + +`--lib --no-default-features`), `cargo test` (both feature configs), +`RUSTDOCFLAGS=-D warnings cargo doc`, `cargo bench --no-run`; the +`tests/cow_snapshot.rs` differential-equivalence gate and the existing +snapshot/overlay/freshness tests pass unchanged. + +Landed on `phase-5-cow-snapshots`: the memoized two-tier base (`BaseState` + +the rewritten two-tier `EvmSnapshot` with `account_info`/`storage_value`/`code` +accessors, `src/cache/snapshot.rs`); `EvmCache::refresh_base`/`build_base_full`, +the COW `create_snapshot` (now `&mut self`), the retained +`create_snapshot_deep_clone`, and the `mark_base_dirty`/`invalidate_base` +invalidation wired into `write_slot_through`/`apply_slot_run`/ +`write_account_info_through`/`inject_storage_batch`/the `purge_*` paths and +`set_block` (`src/cache/mod.rs`); `EvmOverlay::reset` plus the reusable +shared-memory buffer recycled across the call methods (`src/cache/overlay.rs`); +the layer-2-seeded A/B + hot-loop + `reset()`-fanout benches +(`benches/simulation.rs`); and the differential-equivalence gate +(`tests/cow_snapshot.rs`). The residual O(accounts) length-scan / O(layer-1) +fold cost model is recorded in `KNOWN_ISSUES.md`. + +--- + ## Key abstractions for later phases (sketches) ```rust diff --git a/src/cache/mod.rs b/src/cache/mod.rs index 7387e82..9a1e2d9 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -432,6 +432,26 @@ pub struct EvmCache { /// layer hardfork for accurate gas accounting. Configured per-chain via `evm_spec` /// in `chains.toml`. spec_id: SpecId, + /// Memoized, `Arc`-shared flatten of the cold layer-2 index, reused across + /// successive [`create_snapshot`](Self::create_snapshot) calls (Pillar A). + /// `None` until the first snapshot. Rebuilt copy-on-write by + /// [`refresh_base`](Self::refresh_base); never mutated in place once shared. + /// Not part of any public API and not serialized. + base: Option>, + /// Layer-2 addresses changed since `base` was built, folded into the next base + /// rebuild. Populated by the base-invalidation sites (write-through, batch + /// injects, layer-2 seeding, purges). Not serialized. + base_dirty: HashSet
, + /// When set, the next [`refresh_base`](Self::refresh_base) rebuilds the base + /// from scratch. Set by [`set_block`](Self::set_block) / + /// [`repin_to_block`](Self::repin_to_block), which replace layer 2 wholesale. + /// Not serialized. + base_full_rebuild: bool, + /// Per-account layer-2 slot count at the last base build, used by + /// [`refresh_base`](Self::refresh_base)'s `O(accounts)` length-scan to detect + /// uncontrolled lazy-fetch growth that bypasses the write funnel. Not + /// serialized. + base_storage_lens: HashMap, } /// Outcome of a balance-delta-tracking simulation. @@ -909,6 +929,10 @@ impl EvmCache { batch_block_id, erc20_balance_slots: HashMap::new(), spec_id, + base: None, + base_dirty: HashSet::new(), + base_full_rebuild: false, + base_storage_lens: HashMap::new(), } } @@ -983,6 +1007,10 @@ impl EvmCache { batch_block_id: Arc::new(Mutex::new(block.unwrap_or_default())), erc20_balance_slots: HashMap::new(), spec_id, + base: None, + base_dirty: HashSet::new(), + base_full_rebuild: false, + base_storage_lens: HashMap::new(), } } @@ -1064,6 +1092,19 @@ impl EvmCache { /// consistency model: reads here see only the backend layer, not the /// CacheDB overlay, and any writes performed through it skip the overlay. /// Prefer the higher-level accessors; use with care. + /// + /// # Snapshot base + /// Writing layer 2 directly through this handle also bypasses the memoized + /// copy-on-write snapshot base (Pillar A): an **in-place value overwrite at an + /// unchanged slot count** is invisible to the [`create_snapshot`](Self::create_snapshot) + /// growth scan (which is count/absence-based — the lazily-fetched backend only + /// ever *appends*, so that is sufficient for the supported write paths), and a + /// later `create_snapshot` may reuse a stale base. After a direct layer-2 write + /// through this handle, call + /// [`invalidate_snapshot_base`](Self::invalidate_snapshot_base) (or re-pin via + /// [`set_block`](Self::set_block)) before the next snapshot. Writes via the + /// crate's own mutators (`inject_storage_batch`, `apply_update`, the `inject_*` + /// helpers, the purges) keep the base honest automatically. pub fn blockchain_db(&self) -> &BlockchainDb { &self.blockchain_db } @@ -1074,6 +1115,15 @@ impl EvmCache { /// This exposes an internal and bypasses the cache's two-layer consistency /// model: it reads/fetches directly without consulting the CacheDB overlay. /// Prefer the higher-level accessors; use with care. + /// + /// # Snapshot base + /// `SharedBackend::insert_or_update_storage` / `insert_or_update_address` rewrite + /// layer-2 entries **in place**, which (unlike the append-only lazy fetch) can + /// leave the memoized copy-on-write snapshot base stale at an unchanged slot + /// count. After such a direct write, call + /// [`invalidate_snapshot_base`](Self::invalidate_snapshot_base) before the next + /// [`create_snapshot`](Self::create_snapshot). The lazy RPC fetch path needs no + /// such call (it only ever appends, which the snapshot growth scan catches). pub fn backend(&self) -> &SharedBackend { &self.backend } @@ -1124,15 +1174,26 @@ impl EvmCache { self.storage_batch_fetcher.as_ref() } - /// Inject batch-fetched storage values directly into BlockchainDb. + /// Inject batch-fetched storage values directly into BlockchainDb (layer 2). /// /// This bypasses SharedBackend and makes values available for subsequent /// `storage_ref()` calls and EVM SLOADs. Used after `StorageBatchFetchFn` /// returns results to populate the cache in bulk. - pub fn inject_storage_batch(&self, results: &[(Address, U256, U256)]) { - let mut storage = self.blockchain_db.storage().write(); - for &(addr, slot, value) in results { - storage.entry(addr).or_default().insert(slot, value); + /// + /// Takes `&mut self` (as of Pillar A) so it can mark each touched address dirty + /// for the memoized copy-on-write base; the write itself is still a direct + /// layer-2 backend write. Overwriting an existing slot at an unchanged slot + /// count is invalidated here too, since the `refresh_base` growth scan only + /// catches length changes. + pub fn inject_storage_batch(&mut self, results: &[(Address, U256, U256)]) { + { + let mut storage = self.blockchain_db.storage().write(); + for &(addr, slot, value) in results { + storage.entry(addr).or_default().insert(slot, value); + } + } + for &(addr, _, _) in results { + self.mark_base_dirty(addr); } } @@ -1384,7 +1445,10 @@ impl EvmCache { fn apply_slot_run(&mut self, run: &[StateUpdate], diff: &mut StateDiff) { // Borrow the two layers as disjoint fields: the backend storage guard // (layer 2) held for the whole run, and the overlay accounts map (layer 1, - // lock-free). + // lock-free). Base invalidation is deferred until after the guard is + // dropped (it needs `&mut self`): collect the layer-2 addresses written + // here and mark them dirty below. + let mut dirtied: Vec
= Vec::new(); let overlay = &mut self.db.cache.accounts; let mut storage = self.blockchain_db.storage().write(); @@ -1425,6 +1489,9 @@ impl EvmCache { }; write_slot_into(overlay, &mut storage, address, slot, new); + // Layer 2 was written for this address → it must be re-folded into the + // memoized base. Mirrors `write_slot_through`'s `mark_base_dirty`. + dirtied.push(address); if old != new { diff.slots.push(SlotChange { address, @@ -1434,6 +1501,12 @@ impl EvmCache { }); } } + + // Drop the storage write-guard before taking `&mut self` for invalidation. + drop(storage); + for address in dirtied { + self.mark_base_dirty(address); + } } /// Write-through a single storage slot (§5.1). Returns a [`SlotChange`] iff @@ -1475,6 +1548,10 @@ impl EvmCache { if let Some(db_account) = self.db.cache.accounts.get_mut(&address) { db_account.storage.insert(slot, value); } + + // Layer 2 changed → invalidate the memoized base for this address (D2: + // over-invalidation when also shadowed by layer 1 is safe). + self.mark_base_dirty(address); } /// Read-modify-write one storage slot through a caller-supplied transform. @@ -1661,6 +1738,9 @@ impl EvmCache { if overlay_present { self.db.insert_account_info(address, info); } + // Layer-2 account info changed → invalidate the memoized base for this + // address (D2: over-invalidation when also in layer 1 is safe). + self.mark_base_dirty(address); } /// Apply a partial [`AccountPatch`] write-through (§5.2). Returns an @@ -1938,6 +2018,8 @@ impl EvmCache { "purged account from both cache layers" ); } + // Layer 2 (account + storage) changed for this address → invalidate base. + self.mark_base_dirty(addr); (slots_removed, account_removed) } @@ -1997,12 +2079,259 @@ impl EvmCache { /// /// For cheap same-thread save/restore of just the overlay, prefer /// [`snapshot`](Self::snapshot) / [`restore`](Self::restore) instead. - pub fn create_snapshot(&self) -> Arc { + pub fn create_snapshot(&mut self) -> Arc { + // 1. Refresh / memoize the cold layer-2 base, then take a cheap Arc handle + // (O(1) when layer 2 is unchanged since the last snapshot). + self.refresh_base(); + let base = Arc::clone(self.base.as_ref().expect("refresh_base sets base")); + + // 2. Fold layer 1 (the hot CacheDB overlay) into the snapshot's overlay + // maps + cleared/not-existing sets, applying the same classification as + // the legacy flatten (O(layer-1)). + let mut overlay_accounts = HashMap::new(); + let mut overlay_storage = HashMap::new(); + let mut overlay_code_by_hash = HashMap::new(); + let mut storage_cleared = std::collections::HashSet::new(); + let mut accounts_not_existing = std::collections::HashSet::new(); + for (addr, db_account) in &self.db.cache.accounts { + let not_existing = matches!(db_account.account_state, AccountState::NotExisting); + let cleared = + not_existing || matches!(db_account.account_state, AccountState::StorageCleared); + + // Account info. Mirror revm `DbAccount::info()` / `loaded_account_info`: + // a NotExisting overlay account is absent to the EVM (`basic` returns + // None), so it must NOT contribute info/code to the overlay — and + // `accounts_not_existing` makes the read short-circuit to None before + // ever consulting the base. + if not_existing { + accounts_not_existing.insert(*addr); + } else { + if let Some(code) = &db_account.info.code { + overlay_code_by_hash.insert(db_account.info.code_hash, code.clone()); + } + overlay_accounts.insert(*addr, db_account.info.clone()); + } + + // Storage. A StorageCleared/NotExisting account's storage is locally + // complete: the overlay holds ONLY its own slots (so a cleared account + // ALWAYS gets an `overlay_storage` entry, possibly empty), an absent + // slot reads ZERO via `storage_cleared`, and the base is never consulted + // for it. A non-cleared overlay account contributes its slots; absent + // slots fall through to the base on a read. + if cleared { + storage_cleared.insert(*addr); + let account_storage: HashMap = + db_account.storage.iter().map(|(k, v)| (*k, *v)).collect(); + overlay_storage.insert(*addr, account_storage); + } else if !db_account.storage.is_empty() { + let account_storage = overlay_storage.entry(*addr).or_default(); + for (slot, value) in &db_account.storage { + account_storage.insert(*slot, *value); + } + } + } + + Arc::new(snapshot::EvmSnapshot { + base, + overlay_accounts, + overlay_storage, + overlay_code_by_hash, + storage_cleared, + accounts_not_existing, + block_hashes: HashMap::new(), + block_number: self.block_number, + basefee: self.basefee, + coinbase: self.coinbase, + prevrandao: self.prevrandao, + gas_limit: self.block_gas_limit, + chain_id: self.chain_id, + timestamp: self.timestamp_override, + spec_id: self.spec_id, + }) + } + + /// Force the next [`create_snapshot`](Self::create_snapshot) to rebuild the + /// memoized copy-on-write base from scratch (Pillar A). + /// + /// The crate's own mutators keep the base honest automatically. This is the + /// **escape-hatch re-honest hook**: call it after writing layer 2 directly + /// through [`blockchain_db`](Self::blockchain_db) or + /// [`backend`](Self::backend) — those bypass the write funnel, and an in-place + /// value overwrite at an unchanged slot count is invisible to the snapshot + /// growth scan (it is count/absence-based, which suffices for the append-only + /// lazy-fetch path but not for an out-of-band overwrite). Calling this before + /// the next snapshot guarantees it reflects the direct write rather than a + /// stale memoized value. Over-invalidation is always safe (Decision D2); the + /// only cost is one full base rebuild on the next snapshot. + pub fn invalidate_snapshot_base(&mut self) { + self.invalidate_base(); + } + + /// Refresh the memoized cold layer-2 [`BaseState`](snapshot::BaseState), + /// reusing the previous `Arc` wherever layer 2 is unchanged (Pillar A). + /// + /// Called at the top of [`create_snapshot`](Self::create_snapshot). It never + /// mutates an `Arc` that may already be shared with a live + /// snapshot: on any change it builds a *new* `BaseState` that shares the `Arc` + /// handles of unchanged accounts and rebuilds only the changed ones + /// (copy-on-write). + /// + /// Algorithm (see `docs/phase-5-spec.md` §2.3): + /// 1. **Full rebuild** when there is no base yet or `base_full_rebuild` is set + /// (`set_block` / re-pin replaced layer 2): flatten all of layer 2. + /// 2. **Detect uncontrolled growth**: a lazy RPC fetch / prefetch can write + /// layer 2 from inside `foundry-fork-db`, bypassing our write funnel. An + /// `O(accounts)` length-scan over the current layer-2 storage/accounts marks + /// any address whose slot count differs from the recorded length, or any + /// account absent from the base, as dirty. + /// 3. **Nothing dirty** → reuse the existing `Arc` unchanged (the + /// common hot-loop case; the base side of `create_snapshot` is then O(1)). + /// 4. **Some addresses dirty** → build a new `BaseState` sharing the `Arc`s of + /// unchanged accounts and rebuilding only the dirty ones. + fn refresh_base(&mut self) { + // Case 1: full rebuild. + if self.base.is_none() || self.base_full_rebuild { + self.base = Some(Arc::new(self.build_base_full())); + self.base_dirty.clear(); + self.base_full_rebuild = false; + return; + } + + // Case 2: detect uncontrolled layer-2 growth via an O(accounts) length scan + // (NOT an O(slots) value scan). Any address whose slot count changed, or any + // account that newly appeared in layer 2, is folded into `base_dirty`. + // + // LOAD-BEARING INVARIANT: the count/absence scan is sufficient *only* because + // the one uncontrolled layer-2 writer — the foundry-fork-db `SharedBackend` + // lazy fetch — is append-only at a fixed block (its request handler answers an + // already-cached account/slot from the store and only inserts on a miss; it + // never overwrites an existing entry in place). So an uncontrolled fetch can + // only add a new account (caught by the absence check) or a new slot (caught + // by the count check). An in-place value overwrite at unchanged length is + // invisible here; the controlled writers therefore call `mark_base_dirty` + // explicitly, and a direct out-of-band write via `blockchain_db()`/`backend()` + // must call `invalidate_snapshot_base`. If a future foundry-fork-db bump makes + // the lazy path overwrite-in-place, this scan must gain a value/version check. + { + let db_storage = self.blockchain_db.storage().read(); + for (addr, slots) in db_storage.iter() { + if self.base_storage_lens.get(addr).copied() != Some(slots.len()) { + self.base_dirty.insert(*addr); + } + } + let db_accounts = self.blockchain_db.accounts().read(); + let base = self.base.as_ref().expect("base present in case 2/3/4"); + for addr in db_accounts.keys() { + if !base.accounts.contains_key(addr) { + self.base_dirty.insert(*addr); + } + } + } + + // Case 3: nothing changed → reuse the existing Arc unchanged. + if self.base_dirty.is_empty() { + return; + } + + // Case 4: rebuild copy-on-write — clone the outer maps (Arc handles + + // AccountInfo, no per-slot copy) and rebuild only the dirty addresses. + let prev = self.base.as_ref().expect("base present in case 4"); + let mut accounts = prev.accounts.clone(); + let mut storage = prev.storage.clone(); + let mut code_by_hash = prev.code_by_hash.clone(); + + let db_accounts = self.blockchain_db.accounts().read(); + let db_storage = self.blockchain_db.storage().read(); + for addr in self.base_dirty.iter().copied() { + // Account info + code: refresh from the current layer-2 account, or drop + // it if the account no longer exists in layer 2 (e.g. after a purge). + match db_accounts.get(&addr) { + Some(info) => { + if let Some(code) = &info.code { + code_by_hash.insert(info.code_hash, code.clone()); + } + accounts.insert(addr, info.clone()); + } + None => { + accounts.remove(&addr); + } + } + + // Storage: rebuild this account's Arc from the current layer-2 + // storage, or drop it if the account has no layer-2 storage anymore. + match db_storage.get(&addr) { + Some(slots) => { + let rebuilt: HashMap = + slots.iter().map(|(k, v)| (*k, *v)).collect(); + self.base_storage_lens.insert(addr, rebuilt.len()); + storage.insert(addr, Arc::new(rebuilt)); + } + None => { + storage.remove(&addr); + self.base_storage_lens.remove(&addr); + } + } + } + + self.base = Some(Arc::new(snapshot::BaseState { + accounts, + storage, + code_by_hash, + })); + self.base_dirty.clear(); + } + + /// Build a fresh [`BaseState`](snapshot::BaseState) by flattening all of layer + /// 2, recording `base_storage_lens`. Shared by `refresh_base`'s full-rebuild + /// path and [`create_snapshot_deep_clone`](Self::create_snapshot_deep_clone). + fn build_base_full(&mut self) -> snapshot::BaseState { let mut accounts = HashMap::new(); + let mut code_by_hash = HashMap::new(); + { + let db_accounts = self.blockchain_db.accounts().read(); + for (addr, info) in db_accounts.iter() { + if let Some(code) = &info.code { + code_by_hash.insert(info.code_hash, code.clone()); + } + accounts.insert(*addr, info.clone()); + } + } let mut storage = HashMap::new(); + self.base_storage_lens.clear(); + { + let db_storage = self.blockchain_db.storage().read(); + for (addr, slots) in db_storage.iter() { + let converted: HashMap = slots.iter().map(|(k, v)| (*k, *v)).collect(); + self.base_storage_lens.insert(*addr, converted.len()); + storage.insert(*addr, Arc::new(converted)); + } + } + snapshot::BaseState { + accounts, + storage, + code_by_hash, + } + } + + /// The retained deep-clone snapshot — today's full flatten, kept reachable for + /// A/B benchmarking and as the read-equivalence reference (Decision D3). + /// + /// Produces the same two-tier [`EvmSnapshot`](snapshot::EvmSnapshot) shape as + /// [`create_snapshot`](Self::create_snapshot), but with `base` set to the + /// fully-merged flatten of **both** layers and **empty** overlay maps (the + /// cleared / not-existing sets still in place). It is read-indistinguishable + /// from `create_snapshot` by construction (the `tests/cow_snapshot.rs` + /// differential gate pins this), at the cost of an O(total state) deep copy + /// every call — exactly the cost `create_snapshot` now amortizes away. + /// + /// Stays `&self`: it does not touch the memoized base. + #[doc(hidden)] + pub fn create_snapshot_deep_clone(&self) -> Arc { + let mut accounts = HashMap::new(); + let mut storage: HashMap> = HashMap::new(); let mut code_by_hash = HashMap::new(); - // 1. Load from BlockchainDb (persistent cache / Layer 2) + // 1. Load from BlockchainDb (persistent cache / Layer 2). { let db_accounts = self.blockchain_db.accounts().read(); for (addr, info) in db_accounts.iter() { @@ -2015,13 +2344,19 @@ impl EvmCache { { let db_storage = self.blockchain_db.storage().read(); for (addr, slots) in db_storage.iter() { - // Convert from DefaultHashBuilder to RandomState HashMap let converted: HashMap = slots.iter().map(|(k, v)| (*k, *v)).collect(); storage.insert(*addr, converted); } } - // 2. Overlay from CacheDB (Layer 1, takes precedence) + // 2. Overlay from CacheDB (Layer 1, takes precedence). Merge into the same + // flat maps, dropping shadowed entries, exactly as the original + // `create_snapshot` did. A cleared account's storage is routed into + // `overlay_storage` (not the base), because `EvmSnapshot::storage_value` + // only applies the cleared-as-ZERO rule for an address with an + // `overlay_storage` entry — so the cleared semantics must be expressed + // there for both snapshot constructors to read identically. + let mut overlay_storage: HashMap> = HashMap::new(); let mut storage_cleared = std::collections::HashSet::new(); let mut accounts_not_existing = std::collections::HashSet::new(); for (addr, db_account) in &self.db.cache.accounts { @@ -2029,11 +2364,6 @@ impl EvmCache { let cleared = not_existing || matches!(db_account.account_state, AccountState::StorageCleared); - // Account info. Mirror revm `DbAccount::info()` / `loaded_account_info`: - // a NotExisting overlay account is absent to the EVM (`basic` returns - // None), so it must NOT contribute info/code to the snapshot — and any - // backend-merged entry from step 1 is dropped, since loaded_account_info - // short-circuits to None before consulting the backend. if not_existing { accounts_not_existing.insert(*addr); accounts.remove(addr); @@ -2044,17 +2374,16 @@ impl EvmCache { accounts.insert(*addr, db_account.info.clone()); } - // Storage. A StorageCleared/NotExisting account's storage is locally - // complete: the snapshot holds ONLY its overlay slots (any shadowed - // backend slots are dropped) and an absent slot reads ZERO via - // `storage_cleared`, rather than falling through to the (shadowed) - // backend or an ext_db. if cleared { + // Cleared: storage is locally complete. Drop any shadowed base + // slots and keep ONLY the overlay slots, in `overlay_storage`. storage_cleared.insert(*addr); + storage.remove(addr); let account_storage: HashMap = db_account.storage.iter().map(|(k, v)| (*k, *v)).collect(); - storage.insert(*addr, account_storage); + overlay_storage.insert(*addr, account_storage); } else { + // Non-cleared: overlay slots win over base; fold them into base. let account_storage = storage.entry(*addr).or_default(); for (slot, value) in &db_account.storage { account_storage.insert(*slot, *value); @@ -2062,13 +2391,23 @@ impl EvmCache { } } - Arc::new(snapshot::EvmSnapshot { + let base = snapshot::BaseState { accounts, - storage, + storage: storage + .into_iter() + .map(|(addr, slots)| (addr, Arc::new(slots))) + .collect(), + code_by_hash, + }; + + Arc::new(snapshot::EvmSnapshot { + base: Arc::new(base), + overlay_accounts: HashMap::new(), + overlay_storage, + overlay_code_by_hash: HashMap::new(), storage_cleared, accounts_not_existing, block_hashes: HashMap::new(), - code_by_hash, block_number: self.block_number, basefee: self.basefee, coinbase: self.coinbase, @@ -2080,6 +2419,28 @@ impl EvmCache { }) } + /// Mark a layer-2 address dirty so the next [`refresh_base`](Self::refresh_base) + /// re-folds it into the memoized base (Pillar A invalidation; see + /// `docs/phase-5-spec.md` §3). + /// + /// Called from every site that can change a layer-2 value a snapshot read + /// would surface (write-through, batch injects, layer-2 seeding, purges). + /// Over-invalidation is safe (Decision D2): marking an address that is also + /// shadowed by layer 1 just re-folds that one account. + fn mark_base_dirty(&mut self, address: Address) { + self.base_dirty.insert(address); + } + + /// Force a full rebuild of the memoized base on the next + /// [`refresh_base`](Self::refresh_base) (Pillar A invalidation). + /// + /// Used by layer-2 changes too broad to enumerate per-address efficiently + /// (multi-contract / full-storage purges, block re-pins). Coarser than + /// [`mark_base_dirty`](Self::mark_base_dirty) but always correct. + fn invalidate_base(&mut self) { + self.base_full_rebuild = true; + } + /// Update the block that RPC fetches are pinned to. /// /// This re-pins the SharedBackend and the batch storage fetcher to `block`, @@ -2102,6 +2463,9 @@ impl EvmCache { pub fn set_block(&mut self, block: Option) { if self.block != block { self.block = block; + // Re-pinning replaces layer 2 wholesale (state at a new block): the + // memoized base must be rebuilt from scratch on the next snapshot. + self.invalidate_base(); if let Some(block_id) = block { let _ = self.backend.set_pinned_block(block_id); *self.batch_block_id.lock().unwrap() = block_id; @@ -2949,11 +3313,13 @@ impl EvmCache { }; // Layer 2: Clear BlockchainDb backend - let mut storage = self.blockchain_db.storage().write(); - let backend_cleared = if let Some(slots) = storage.remove(&address) { - slots.len() - } else { - 0 + let backend_cleared = { + let mut storage = self.blockchain_db.storage().write(); + if let Some(slots) = storage.remove(&address) { + slots.len() + } else { + 0 + } }; if cache_db_cleared > 0 || backend_cleared > 0 { @@ -2965,6 +3331,8 @@ impl EvmCache { ); } + // Layer-2 storage for this address was removed → invalidate base. + self.mark_base_dirty(address); backend_cleared } @@ -3006,11 +3374,13 @@ impl EvmCache { } // Layer 2: Remove specific slots from BlockchainDb backend - let mut storage = self.blockchain_db.storage().write(); - if let Some(address_storage) = storage.get_mut(&address) { - for slot in slots { - if address_storage.remove(slot).is_some() { - backend_removed += 1; + { + let mut storage = self.blockchain_db.storage().write(); + if let Some(address_storage) = storage.get_mut(&address) { + for slot in slots { + if address_storage.remove(slot).is_some() { + backend_removed += 1; + } } } } @@ -3025,6 +3395,9 @@ impl EvmCache { ); } + // Layer-2 storage for this address changed (slots dropped) → invalidate + // base. The growth scan only catches length changes; mark explicitly. + self.mark_base_dirty(address); backend_removed } @@ -3064,6 +3437,9 @@ impl EvmCache { "purged contract storage from both cache layers" ); } + // Multiple layer-2 contracts changed → full base rebuild (coarse but + // correct; cheaper than enumerating each touched address here). + self.invalidate_base(); total_purged } @@ -3092,10 +3468,13 @@ impl EvmCache { } // Layer 2: Clear BlockchainDb backend - let mut storage = self.blockchain_db.storage().write(); - let total_slots: usize = storage.values().map(|s| s.len()).sum(); - let contract_count = storage.len(); - storage.clear(); + let (total_slots, contract_count) = { + let mut storage = self.blockchain_db.storage().write(); + let total_slots: usize = storage.values().map(|s| s.len()).sum(); + let contract_count = storage.len(); + storage.clear(); + (total_slots, contract_count) + }; if total_slots > 0 || cache_db_cleared > 0 { warn!( @@ -3105,6 +3484,8 @@ impl EvmCache { "purged ALL storage from both cache layers (full refresh)" ); } + // All layer-2 storage was cleared → full base rebuild. + self.invalidate_base(); total_slots } @@ -3484,6 +3865,12 @@ impl EvmCache { accounts.insert(target, target_info); } + // Layer 2 changed → invalidate the memoized base for `target`. The layer-1 + // `insert_account_info` above currently shadows it on every snapshot read, + // but we dirty unconditionally for uniformity with every other layer-2 write + // site (D2), so base correctness never relies on that shadowing invariant. + self.mark_base_dirty(target); + Ok(()) } diff --git a/src/cache/overlay.rs b/src/cache/overlay.rs index 7656e22..4e3fe38 100644 --- a/src/cache/overlay.rs +++ b/src/cache/overlay.rs @@ -40,6 +40,13 @@ type InspectorOverlayEvm<'a, INSP> = revm::MainnetEvm< /// This type is `Send` (unlike `EvmCache`) because it uses no `Rc`/`RefCell`. /// Each simulation task gets its own `EvmOverlay` with a cheap `Arc::clone` /// of the shared `EvmSnapshot`. +/// +/// # Reuse across simulations (Pillar A.2) +/// +/// A worker doing many sims against the same snapshot can call [`Self::new`] +/// once and [`Self::reset`] between sims instead of allocating a fresh overlay +/// each time. The reusable shared-memory buffer is also recycled across calls — +/// see [`Self::call_raw`] — without making the overlay `!Send`. pub struct EvmOverlay { snapshot: Arc, /// Per-simulation mutations (accounts fetched from ext_db, committed changes). @@ -48,6 +55,14 @@ pub struct EvmOverlay { dirty_storage: HashMap>, /// Optional RPC fallback for data not in snapshot. ext_db: Option, + /// Reusable shared-memory buffer, recycled across the build→transact→revert + /// call methods to avoid reallocating a 64 KB `Vec` per call. + /// + /// Stored as a plain `Vec` (not an `Rc`) so the overlay stays `Send`. A + /// call method `mem::take`s it, wraps it in a method-local `Rc>` + /// for revm's [`LocalContext`], runs, then reclaims and clears it after the + /// EVM is dropped (see [`Self::build_evm_with_local`]). + reusable_buffer: Vec, } impl EvmOverlay { @@ -58,9 +73,27 @@ impl EvmOverlay { dirty_accounts: HashMap::new(), dirty_storage: HashMap::new(), ext_db, + reusable_buffer: Vec::with_capacity(OVERLAY_SHARED_MEMORY_CAPACITY), } } + /// Clear the per-simulation dirty layer so this overlay can be reused for the + /// next simulation against the same snapshot, without reallocating (Pillar + /// A.2). + /// + /// A worker doing K sims calls [`Self::new`] once and `reset()` between sims + /// instead of allocating a fresh overlay (plus dirty maps plus an `Arc` + /// clone) each time. After `reset()` the overlay reads the pristine snapshot + /// again — it is exactly equivalent to a freshly-built overlay on the same + /// snapshot. The snapshot `Arc`, the optional `ext_db`, and the reusable + /// shared-memory buffer (kept at capacity) are retained. + pub fn reset(&mut self) { + self.dirty_accounts.clear(); + self.dirty_storage.clear(); + // Keep: snapshot Arc, ext_db, and the reusable buffer. The buffer is + // already cleared after each call, so nothing to do for it here. + } + /// Chain ID of the block context captured by the underlying snapshot. /// /// This is the value installed into `cfg.chain_id` by [`Self::build_evm`]. @@ -95,17 +128,31 @@ impl EvmOverlay { self.snapshot.timestamp } - /// Build a revm EVM instance backed by this overlay. + /// A fresh [`LocalContext`] with a newly-allocated 64 KB shared-memory buffer. /// - /// Note: The returned EVM is `!Send` (due to `LocalContext`'s `Rc`), - /// but this is fine because it's created and used within a single task. - pub fn build_evm(&mut self) -> OverlayEvm<'_> { - let local = LocalContext { + /// Used by the public [`Self::build_evm`], which hands out the EVM and cannot + /// reclaim its buffer afterwards. The internal call methods instead recycle + /// [`Self::reusable_buffer`] via [`Self::build_evm_with_local`]. + fn fresh_local() -> LocalContext { + LocalContext { shared_memory_buffer: Rc::new(RefCell::new(Vec::with_capacity( OVERLAY_SHARED_MEMORY_CAPACITY, ))), precompile_error_message: None, - }; + } + } + + /// Build a revm EVM instance backed by this overlay, using a caller-supplied + /// [`LocalContext`]. + /// + /// This is the shared body behind [`Self::build_evm`] and the internal call + /// methods. The call methods pass a `local` wrapping the recycled + /// [`Self::reusable_buffer`] (Pillar A.2) and reclaim it after the EVM is + /// dropped; [`Self::build_evm`] passes a fresh one. + /// + /// Note: the returned EVM is `!Send` (due to `LocalContext`'s `Rc`), + /// but this is fine because it's created and used within a single task. + fn build_evm_with_local(&mut self, local: LocalContext) -> OverlayEvm<'_> { // Read snapshot values before the mutable borrow of self let chain_id = self.snapshot.chain_id; let spec_id = self.snapshot.spec_id; @@ -155,6 +202,20 @@ impl EvmOverlay { evm } + /// Build a revm EVM instance backed by this overlay. + /// + /// This allocates a fresh 64 KB shared-memory buffer each call: it hands the + /// EVM out to the caller and cannot reclaim the buffer afterwards, so it + /// cannot recycle the overlay's reusable buffer. The internal call methods + /// ([`Self::call_raw`], etc.) recycle the buffer instead (Pillar A.2). + /// + /// Note: The returned EVM is `!Send` (due to `LocalContext`'s `Rc`), + /// but this is fine because it's created and used within a single task. + pub fn build_evm(&mut self) -> OverlayEvm<'_> { + let local = Self::fresh_local(); + self.build_evm_with_local(local) + } + /// Execute a non-committing call and return the raw [`ExecutionResult`]. /// /// The EVM state is reverted to a checkpoint after execution on *both* @@ -201,24 +262,59 @@ impl EvmOverlay { .build() .map_err(|e| anyhow!("Failed to build tx env: {:?}", e))?; - let mut evm = self.build_evm(); - use revm::context_interface::JournalTr; - let checkpoint = evm.journaled_state.checkpoint(); - let result = evm - .transact_one(tx) - .map_err(|e| anyhow!("Failed to transact: {:?}", e)); - evm.journaled_state.checkpoint_revert(checkpoint); - result - } - - /// Build a revm EVM instance with an inspector, backed by this overlay. - fn build_evm_with_inspector(&mut self, inspector: INSP) -> InspectorOverlayEvm<'_, INSP> { + // Recycle the reusable buffer (Pillar A.2): take it out as a plain Vec + // (keeping the overlay Send), lend it to a method-local Rc for + // revm's LocalContext, then reclaim and clear it after the EVM is dropped. + let buffer = Rc::new(RefCell::new(std::mem::take(&mut self.reusable_buffer))); let local = LocalContext { - shared_memory_buffer: Rc::new(RefCell::new(Vec::with_capacity( - OVERLAY_SHARED_MEMORY_CAPACITY, - ))), + shared_memory_buffer: Rc::clone(&buffer), precompile_error_message: None, }; + + let result = { + let mut evm = self.build_evm_with_local(local); + use revm::context_interface::JournalTr; + let checkpoint = evm.journaled_state.checkpoint(); + let result = evm + .transact_one(tx) + .map_err(|e| anyhow!("Failed to transact: {:?}", e)); + evm.journaled_state.checkpoint_revert(checkpoint); + result + }; + + self.reclaim_buffer(buffer); + result + } + + /// Reclaim the recycled shared-memory buffer after the EVM (and its + /// `LocalContext` clone of the `Rc`) has been dropped, clearing it for the + /// next call. + /// + /// The `Rc` was only ever held by the dropped EVM and this method's local, so + /// `try_unwrap` succeeds in the normal path. If a panic somewhere left an + /// extra strong reference the buffer is simply re-allocated next call — no + /// correctness impact. + fn reclaim_buffer(&mut self, buffer: Rc>>) { + if let Ok(cell) = Rc::try_unwrap(buffer) { + let mut buf = cell.into_inner(); + buf.clear(); + self.reusable_buffer = buf; + } else { + self.reusable_buffer = Vec::with_capacity(OVERLAY_SHARED_MEMORY_CAPACITY); + } + } + + /// Build a revm EVM instance with an inspector, backed by this overlay, using + /// a caller-supplied [`LocalContext`]. + /// + /// Like [`Self::build_evm_with_local`] but attaches `inspector`. The call + /// methods pass a `local` wrapping the recycled [`Self::reusable_buffer`] + /// (Pillar A.2) and reclaim it after the EVM is dropped. + fn build_evm_with_inspector_local( + &mut self, + inspector: INSP, + local: LocalContext, + ) -> InspectorOverlayEvm<'_, INSP> { let chain_id = self.snapshot.chain_id; let spec_id = self.snapshot.spec_id; let timestamp = self.snapshot.timestamp.unwrap_or_else(|| { @@ -326,62 +422,75 @@ impl EvmOverlay { .map_err(|e| SimError::Other(anyhow!("Failed to build tx env: {:?}", e)))?; let inspector = TransferInspector::new(); - let mut evm = self.build_evm_with_inspector(inspector); - - use revm::context_interface::JournalTr; - let checkpoint = evm.journaled_state.checkpoint(); - - let result = evm - .inspect_one_tx(tx) - .map_err(|e| SimError::Other(anyhow!("Failed to transact: {:?}", e))); - - match result { - Ok(ExecutionResult::Success { - logs, - gas_used, - output, - .. - }) => { - let token_deltas = if let Some(token_list) = tokens { - evm.inspector.balance_deltas_for_tokens(owner, token_list) - } else { - evm.inspector.balance_deltas(owner) - }; - - // Extract EIP-2930 access list from journaled state - let access_list = extract_access_list(&evm.journaled_state.state); - - if commit { - evm.commit_inner(); - } else { - evm.journaled_state.checkpoint_revert(checkpoint); - } - Ok(CallSimulationResult { - status: SimStatus::Success, - gas_used, - token_deltas, + // Recycle the reusable buffer (Pillar A.2); reclaimed after the EVM drops. + let buffer = Rc::new(RefCell::new(std::mem::take(&mut self.reusable_buffer))); + let local = LocalContext { + shared_memory_buffer: Rc::clone(&buffer), + precompile_error_message: None, + }; + + let outcome = { + let mut evm = self.build_evm_with_inspector_local(inspector, local); + + use revm::context_interface::JournalTr; + let checkpoint = evm.journaled_state.checkpoint(); + + let result = evm + .inspect_one_tx(tx) + .map_err(|e| SimError::Other(anyhow!("Failed to transact: {:?}", e))); + + match result { + Ok(ExecutionResult::Success { logs, - access_list, - output: output.into_data(), - }) - } - Ok(ExecutionResult::Revert { gas_used, output }) => { - evm.journaled_state.checkpoint_revert(checkpoint); - Err(SimulationError::from_revert(gas_used, output).into()) - } - Ok(ExecutionResult::Halt { reason, gas_used }) => { - evm.journaled_state.checkpoint_revert(checkpoint); - Err(SimError::Halt { - reason: format!("{reason:?}"), gas_used, - }) - } - Err(err) => { - evm.journaled_state.checkpoint_revert(checkpoint); - Err(err) + output, + .. + }) => { + let token_deltas = if let Some(token_list) = tokens { + evm.inspector.balance_deltas_for_tokens(owner, token_list) + } else { + evm.inspector.balance_deltas(owner) + }; + + // Extract EIP-2930 access list from journaled state + let access_list = extract_access_list(&evm.journaled_state.state); + + if commit { + evm.commit_inner(); + } else { + evm.journaled_state.checkpoint_revert(checkpoint); + } + + Ok(CallSimulationResult { + status: SimStatus::Success, + gas_used, + token_deltas, + logs, + access_list, + output: output.into_data(), + }) + } + Ok(ExecutionResult::Revert { gas_used, output }) => { + evm.journaled_state.checkpoint_revert(checkpoint); + Err(SimulationError::from_revert(gas_used, output).into()) + } + Ok(ExecutionResult::Halt { reason, gas_used }) => { + evm.journaled_state.checkpoint_revert(checkpoint); + Err(SimError::Halt { + reason: format!("{reason:?}"), + gas_used, + }) + } + Err(err) => { + evm.journaled_state.checkpoint_revert(checkpoint); + Err(err) + } } - } + }; + + self.reclaim_buffer(buffer); + outcome } /// Execute a non-committing call and return the result plus the touched @@ -464,30 +573,42 @@ impl EvmOverlay { .build() .map_err(|e| anyhow!("Failed to build tx env: {:?}", e))?; - let mut evm = self.build_evm(); - use revm::context_interface::JournalTr; - let checkpoint = evm.journaled_state.checkpoint(); - match evm.transact_one(tx_env) { - Ok(result) => { - let mut access_list = StorageAccessList::default(); - for (address, account) in evm.journaled_state.state.iter() { - if account.is_touched() { - access_list.accounts.insert(*address); - for (slot_key, _) in account.storage.iter() { - access_list.slots.insert((*address, *slot_key)); + // Recycle the reusable buffer (Pillar A.2); reclaimed after the EVM drops. + let buffer = Rc::new(RefCell::new(std::mem::take(&mut self.reusable_buffer))); + let local = LocalContext { + shared_memory_buffer: Rc::clone(&buffer), + precompile_error_message: None, + }; + + let outcome = { + let mut evm = self.build_evm_with_local(local); + use revm::context_interface::JournalTr; + let checkpoint = evm.journaled_state.checkpoint(); + match evm.transact_one(tx_env) { + Ok(result) => { + let mut access_list = StorageAccessList::default(); + for (address, account) in evm.journaled_state.state.iter() { + if account.is_touched() { + access_list.accounts.insert(*address); + for (slot_key, _) in account.storage.iter() { + access_list.slots.insert((*address, *slot_key)); + } } } + evm.journaled_state.checkpoint_revert(checkpoint); + Ok((result, access_list)) + } + Err(e) => { + // Revert the checkpoint even on a host/transact error so the EVM + // journal is not left dirty (mirrors `call_raw`). + evm.journaled_state.checkpoint_revert(checkpoint); + Err(anyhow!("Failed to transact: {:?}", e)) } - evm.journaled_state.checkpoint_revert(checkpoint); - Ok((result, access_list)) - } - Err(e) => { - // Revert the checkpoint even on a host/transact error so the EVM - // journal is not left dirty (mirrors `call_raw`). - evm.journaled_state.checkpoint_revert(checkpoint); - Err(anyhow!("Failed to transact: {:?}", e)) } - } + }; + + self.reclaim_buffer(buffer); + outcome } /// Write a storage value into this overlay's dirty layer. @@ -549,16 +670,16 @@ impl Database for EvmOverlay { if let Some(info) = self.dirty_accounts.get(&address) { return Ok(Some(info.clone())); } - // 2. Check snapshot (O(1) HashMap lookup, no locks) - if let Some(info) = self.snapshot.accounts.get(&address) { - return Ok(Some(info.clone())); - } - // 2b. A NotExisting account is absent to the EVM: return None and do NOT - // fall through to the ext_db, mirroring revm `DbAccount::info()` and the - // live `EvmCache` account read (symmetric with `storage_cleared`). + // 2. Check snapshot (O(1) HashMap lookup, no locks). `account_info` folds + // the two snapshot tiers (overlay ▸ base) and already short-circuits a + // NotExisting account to None — it must NOT fall through to the ext_db, + // mirroring revm `DbAccount::info()` and the live `EvmCache` read. if self.snapshot.accounts_not_existing.contains(&address) { return Ok(None); } + if let Some(info) = self.snapshot.account_info(address) { + return Ok(Some(info.clone())); + } // 3. RPC fallback if let Some(ref ext_db) = self.ext_db { let info = ext_db.basic_ref(address)?; @@ -579,8 +700,8 @@ impl Database for EvmOverlay { return Ok(code.clone()); } } - // Check snapshot's code_by_hash index - if let Some(code) = self.snapshot.code_by_hash.get(&code_hash) { + // Check the snapshot's code index (overlay ▸ base). + if let Some(code) = self.snapshot.code(code_hash) { return Ok(code.clone()); } // RPC fallback @@ -597,17 +718,12 @@ impl Database for EvmOverlay { { return Ok(*value); } - // 2. Check snapshot (O(1)) - if let Some(account_storage) = self.snapshot.storage.get(&address) - && let Some(value) = account_storage.get(&index) - { - return Ok(*value); - } - // 2b. A cleared account's storage is locally complete: an absent slot reads - // ZERO and must NOT fall through to the ext_db, mirroring the live EVM - // SLOAD for a StorageCleared/NotExisting account. - if self.snapshot.storage_cleared.contains(&address) { - return Ok(U256::ZERO); + // 2. Check snapshot (O(1)). `storage_value` folds the two tiers (overlay ▸ + // cleared-as-ZERO ▸ base); a cleared account's absent slot reads ZERO + // and must NOT fall through to the ext_db, mirroring the live EVM SLOAD + // for a StorageCleared/NotExisting account. + if let Some(value) = self.snapshot.storage_value(address, index) { + return Ok(value); } // 3. RPC fallback if let Some(ref ext_db) = self.ext_db { @@ -651,7 +767,46 @@ fn extract_access_list(state: &revm::state::EvmState) -> AccessList { #[cfg(test)] mod tests { use super::*; + use crate::cache::snapshot::BaseState; use revm::primitives::hardfork::SpecId; + use std::collections::HashSet; + + /// Build a two-tier `EvmSnapshot` whose cold base holds the given accounts, + /// storage, and code, with an empty hot overlay — the shape + /// `create_snapshot_deep_clone` produces. The `Arc`-per-account storage of the + /// base is built from the plain per-account maps. + fn snap( + accounts: HashMap, + storage: HashMap>, + code_by_hash: HashMap, + block_hashes: HashMap, + ) -> Arc { + let base = BaseState { + accounts, + storage: storage + .into_iter() + .map(|(addr, slots)| (addr, Arc::new(slots))) + .collect(), + code_by_hash, + }; + Arc::new(EvmSnapshot { + base: Arc::new(base), + overlay_accounts: HashMap::new(), + overlay_storage: HashMap::new(), + overlay_code_by_hash: HashMap::new(), + storage_cleared: HashSet::new(), + accounts_not_existing: HashSet::new(), + block_hashes, + block_number: None, + basefee: None, + coinbase: None, + prevrandao: None, + gas_limit: None, + chain_id: 42161, + timestamp: None, + spec_id: SpecId::CANCUN, + }) + } #[test] fn test_overlay_is_send() { @@ -672,22 +827,7 @@ mod tests { let addr = Address::repeat_byte(0x01); accounts.insert(addr, info); - let snapshot = Arc::new(EvmSnapshot { - accounts, - storage: HashMap::new(), - block_hashes: HashMap::new(), - storage_cleared: std::collections::HashSet::new(), - accounts_not_existing: std::collections::HashSet::new(), - code_by_hash: HashMap::new(), - block_number: None, - basefee: None, - coinbase: None, - prevrandao: None, - gas_limit: None, - chain_id: 42161, - timestamp: None, - spec_id: SpecId::CANCUN, - }); + let snapshot = snap(accounts, HashMap::new(), HashMap::new(), HashMap::new()); let mut overlay = EvmOverlay::new(snapshot, None); let result = overlay.basic(addr).unwrap(); @@ -706,22 +846,7 @@ mod tests { account_storage.insert(slot, value); storage.insert(addr, account_storage); - let snapshot = Arc::new(EvmSnapshot { - accounts: HashMap::new(), - storage, - block_hashes: HashMap::new(), - storage_cleared: std::collections::HashSet::new(), - accounts_not_existing: std::collections::HashSet::new(), - code_by_hash: HashMap::new(), - block_number: None, - basefee: None, - coinbase: None, - prevrandao: None, - gas_limit: None, - chain_id: 42161, - timestamp: None, - spec_id: SpecId::CANCUN, - }); + let snapshot = snap(HashMap::new(), storage, HashMap::new(), HashMap::new()); let mut overlay = EvmOverlay::new(snapshot, None); let result = overlay.storage(addr, slot).unwrap(); @@ -738,22 +863,7 @@ mod tests { account_storage.insert(slot, U256::from(100)); storage.insert(addr, account_storage); - let snapshot = Arc::new(EvmSnapshot { - accounts: HashMap::new(), - storage, - block_hashes: HashMap::new(), - storage_cleared: std::collections::HashSet::new(), - accounts_not_existing: std::collections::HashSet::new(), - code_by_hash: HashMap::new(), - block_number: None, - basefee: None, - coinbase: None, - prevrandao: None, - gas_limit: None, - chain_id: 42161, - timestamp: None, - spec_id: SpecId::CANCUN, - }); + let snapshot = snap(HashMap::new(), storage, HashMap::new(), HashMap::new()); let mut overlay = EvmOverlay::new(snapshot, None); @@ -771,22 +881,12 @@ mod tests { #[test] fn test_overlay_missing_returns_zero() { - let snapshot = Arc::new(EvmSnapshot { - accounts: HashMap::new(), - storage: HashMap::new(), - block_hashes: HashMap::new(), - storage_cleared: std::collections::HashSet::new(), - accounts_not_existing: std::collections::HashSet::new(), - code_by_hash: HashMap::new(), - block_number: None, - basefee: None, - coinbase: None, - prevrandao: None, - gas_limit: None, - chain_id: 42161, - timestamp: None, - spec_id: SpecId::CANCUN, - }); + let snapshot = snap( + HashMap::new(), + HashMap::new(), + HashMap::new(), + HashMap::new(), + ); let mut overlay = EvmOverlay::new(snapshot, None); let addr = Address::repeat_byte(0x99); @@ -805,22 +905,7 @@ mod tests { let mut code_by_hash = HashMap::new(); code_by_hash.insert(hash, code.clone()); - let snapshot = Arc::new(EvmSnapshot { - accounts: HashMap::new(), - storage: HashMap::new(), - block_hashes: HashMap::new(), - storage_cleared: std::collections::HashSet::new(), - accounts_not_existing: std::collections::HashSet::new(), - code_by_hash, - block_number: None, - basefee: None, - coinbase: None, - prevrandao: None, - gas_limit: None, - chain_id: 42161, - timestamp: None, - spec_id: SpecId::CANCUN, - }); + let snapshot = snap(HashMap::new(), HashMap::new(), code_by_hash, HashMap::new()); let mut overlay = EvmOverlay::new(snapshot, None); let result = overlay.code_by_hash(hash).unwrap(); @@ -833,22 +918,7 @@ mod tests { let hash = B256::repeat_byte(0xAB); block_hashes.insert(42u64, hash); - let snapshot = Arc::new(EvmSnapshot { - accounts: HashMap::new(), - storage: HashMap::new(), - storage_cleared: std::collections::HashSet::new(), - accounts_not_existing: std::collections::HashSet::new(), - block_hashes, - code_by_hash: HashMap::new(), - block_number: None, - basefee: None, - coinbase: None, - prevrandao: None, - gas_limit: None, - chain_id: 42161, - timestamp: None, - spec_id: SpecId::CANCUN, - }); + let snapshot = snap(HashMap::new(), HashMap::new(), HashMap::new(), block_hashes); let mut overlay = EvmOverlay::new(snapshot, None); assert_eq!(overlay.block_hash(42).unwrap(), hash); diff --git a/src/cache/snapshot.rs b/src/cache/snapshot.rs index c6e8011..3e3e50f 100644 --- a/src/cache/snapshot.rs +++ b/src/cache/snapshot.rs @@ -1,19 +1,28 @@ //! Immutable, shareable EVM state snapshots. //! -//! # Flattening model +//! # Two-tier copy-on-write model (Pillar A) //! -//! A snapshot flattens the live cache (CacheDB overlay plus the BlockchainDb -//! backend) into a single immutable, `Send + Sync` view of accounts and -//! storage. The layered lookups of the live cache are collapsed into flat -//! `HashMap`s at creation time, so every read against the snapshot is an O(1) -//! lookup with no locks and no fallback chain. +//! A snapshot is split into two tiers: //! -//! # `Arc` sharing +//! - a **memoized immutable base** (`BaseState`) flattening the *cold* layer-2 +//! `BlockchainDb` index, shared across successive snapshots by `Arc` — both the +//! base as a whole and each account's storage map (`Arc>`) — +//! so taking a snapshot when the cold index is unchanged is an `Arc` handle +//! copy, never a per-slot deep copy; +//! - a small per-snapshot **overlay** folding the *hot* layer-1 CacheDB delta +//! (committed sim changes, write-throughs, freshness corrections), which always +//! shadows the base on a read. //! -//! Because the snapshot is read-only it can be wrapped in an `Arc` and shared -//! across threads, letting many parallel simulations read from one consistent -//! state. Handing a new simulation task its state is a cheap `Arc::clone` -//! rather than a deep copy of the accounts/storage maps. +//! [`super::EvmCache::create_snapshot`] memoizes the base (via the internal +//! `refresh_base`) and folds only layer 1 fresh, so its cost tracks *changed* +//! state, not *total* state. The retained +//! [`super::EvmCache::create_snapshot_deep_clone`] produces the same two-tier +//! shape with everything flattened into the base and empty overlay maps; it is the +//! A/B benchmark baseline and the read-equivalence reference. +//! +//! Reads stay O(1) `HashMap` lookups with no locks (Decision D1: `Arc` sharing, +//! not a persistent/HAMT map), so the snapshot is `Send + Sync` and an +//! [`EvmOverlay`] built from it is `Send`. //! //! # Per-simulation dirty layer //! @@ -29,35 +38,71 @@ //! [`EvmOverlay`]: super::EvmOverlay use std::collections::{HashMap, HashSet}; +use std::sync::Arc; use alloy_primitives::{Address, B256, U256}; use revm::primitives::hardfork::SpecId; use revm::state::{AccountInfo, Bytecode}; +/// Memoized, immutable flatten of the **cold layer-2** index (Pillar A). +/// +/// Holds layer-2 (`BlockchainDb`) account info and storage only; the layer-1 +/// `StorageCleared` / `NotExisting` classification is purely a layer-1 property +/// and lives on [`EvmSnapshot`], not here (see the read rules on +/// [`EvmSnapshot::storage_value`]). Each account's storage is wrapped in an `Arc` +/// so that rebuilding the base on a partial change (copy-on-write) shares the +/// `Arc` handles of unchanged accounts instead of deep-copying their slots. +/// +/// Built and memoized by [`EvmCache::refresh_base`](super::EvmCache::refresh_base); +/// shared across snapshots and across threads via `Arc`. +pub(crate) struct BaseState { + /// Layer-2 account info, by address. (Layer 2 has no `NotExisting` concept; + /// that classification is purely a layer-1 property — see [`EvmSnapshot`].) + pub(crate) accounts: HashMap, + /// Layer-2 storage, per account, **shared by `Arc`** so cloning a base — or + /// rebuilding it for an unchanged account — is a handle copy, never a per-slot + /// copy. + pub(crate) storage: HashMap>>, + /// Bytecode by `code_hash`, derived from `accounts` at build time. + pub(crate) code_by_hash: HashMap, +} + /// Immutable EVM state snapshot — `Send + Sync`, shared via `Arc` across threads. /// -/// Contains merged account info + storage from both CacheDB overlay and -/// BlockchainDb backend, providing a single flat `HashMap` view for O(1) lookups. +/// A two-tier copy-on-write view (see the [module docs](self)): an `Arc`-shared, +/// memoized cold base (layer 2) plus a small per-snapshot overlay folding the hot +/// layer-1 CacheDB delta, which shadows the base on reads. Lookups (including the +/// public [`storage_value`](Self::storage_value)) are O(1) and lock-free, and +/// reproduce the live cache's layered semantics bit-for-bit. /// /// Created via [`super::EvmCache::create_snapshot()`]. Each parallel simulation /// task gets its own [`super::EvmOverlay`] backed by a cheap `Arc::clone` of /// the snapshot. pub struct EvmSnapshot { - pub(crate) accounts: HashMap, - pub(crate) storage: HashMap>, + /// Memoized, `Arc`-shared cold layer-2 base. + pub(crate) base: Arc, + /// Layer-1 accounts that are present to the EVM (`NotExisting` excluded). + /// Shadows [`BaseState::accounts`] on a read. + pub(crate) overlay_accounts: HashMap, + /// Layer-1 storage delta, per account. A cleared account (revm + /// `StorageCleared` / `NotExisting`) ALWAYS has an entry here (possibly empty) + /// so the cleared rule is decided without consulting the base. + pub(crate) overlay_storage: HashMap>, + /// Bytecode introduced by layer 1 (checked before [`BaseState::code_by_hash`]). + pub(crate) overlay_code_by_hash: HashMap, /// Accounts whose storage is locally complete (revm `StorageCleared` / - /// `NotExisting`): a slot absent from `storage` for such an account reads as - /// ZERO and must NOT fall through to an `ext_db`, mirroring the live EVM SLOAD - /// and [`EvmCache::cached_storage_value`](super::EvmCache::cached_storage_value). + /// `NotExisting`): a slot absent from `overlay_storage` for such an account + /// reads as ZERO and must NOT fall through to the base or an `ext_db`, + /// mirroring the live EVM SLOAD and + /// [`EvmCache::cached_storage_value`](super::EvmCache::cached_storage_value). pub(crate) storage_cleared: HashSet
, - /// Accounts that are absent to the EVM (revm `NotExisting`): `basic` returns - /// `None` for them and must NOT fall through to an `ext_db`, mirroring revm - /// `DbAccount::info()` and [`EvmCache`](super::EvmCache)'s live account read. - /// These addresses are excluded from `accounts` / `code_by_hash`. + /// Accounts that are absent to the EVM (revm `NotExisting`): + /// [`account_info`](Self::account_info) returns `None` for them and must NOT + /// fall through to the base or an `ext_db`, mirroring revm `DbAccount::info()` + /// and [`EvmCache`](super::EvmCache)'s live account read. These addresses are + /// excluded from `overlay_accounts` / `overlay_code_by_hash`. pub(crate) accounts_not_existing: HashSet
, pub(crate) block_hashes: HashMap, - /// Bytecode lookup by code_hash (derived from accounts at creation time). - pub(crate) code_by_hash: HashMap, // Block context pub(crate) block_number: Option, pub(crate) basefee: Option, @@ -70,33 +115,67 @@ pub struct EvmSnapshot { } impl EvmSnapshot { + /// Account info as the EVM sees it: overlay (layer 1) wins, else the base + /// (layer 2), else `None`. + /// + /// Returns `None` for a `NotExisting` account without consulting the base, + /// mirroring revm `DbAccount::info()` and the live `EvmCache` account read. + pub(crate) fn account_info(&self, address: Address) -> Option<&AccountInfo> { + if self.accounts_not_existing.contains(&address) { + return None; + } + self.overlay_accounts + .get(&address) + .or_else(|| self.base.accounts.get(&address)) + } + /// Return the snapshot's value for a storage slot, mirroring the live read. /// /// Used by the freshness validator to compare a freshly-fetched value against /// the value the snapshot was built from. Resolution matches - /// [`EvmCache::cached_storage_value`](super::EvmCache::cached_storage_value): - /// a captured slot returns its value; a slot absent from a cleared account - /// (revm `StorageCleared`/`NotExisting`) returns `Some(ZERO)` (its storage is - /// locally complete); any other absent slot returns `None`. + /// [`EvmCache::cached_storage_value`](super::EvmCache::cached_storage_value) + /// over the two tiers: an overlay (layer-1) slot wins; for a cleared account + /// an absent overlay slot returns `Some(ZERO)` (its storage is locally + /// complete — the base is never consulted); otherwise the base (layer-2) slot + /// is returned, or `None` if neither tier has seen the slot. pub fn storage_value(&self, address: Address, slot: U256) -> Option { - if let Some(value) = self + if let Some(account_storage) = self.overlay_storage.get(&address) { + if let Some(value) = account_storage.get(&slot) { + return Some(*value); + } + // A StorageCleared / NotExisting account's storage is locally complete: + // an absent slot reads ZERO and never falls through to the base. + if self.storage_cleared.contains(&address) { + return Some(U256::ZERO); + } + // Non-cleared overlay account: fall through to the base below. + } + self.base .storage .get(&address) .and_then(|s| s.get(&slot).copied()) - { - return Some(value); - } - if self.storage_cleared.contains(&address) { - return Some(U256::ZERO); - } - None + } + + /// Bytecode by `code_hash`: overlay (layer 1) wins, else the base (layer 2). + pub(crate) fn code(&self, code_hash: B256) -> Option<&Bytecode> { + self.overlay_code_by_hash + .get(&code_hash) + .or_else(|| self.base.code_by_hash.get(&code_hash)) } } #[cfg(test)] mod tests { use super::*; - use std::sync::Arc; + + /// Build an empty `Arc` for snapshot literals in tests. + fn empty_base() -> Arc { + Arc::new(BaseState { + accounts: HashMap::new(), + storage: HashMap::new(), + code_by_hash: HashMap::new(), + }) + } #[test] fn test_snapshot_is_send_sync() { @@ -108,12 +187,13 @@ mod tests { #[test] fn test_empty_snapshot() { let snap = EvmSnapshot { - accounts: HashMap::new(), - storage: HashMap::new(), + base: empty_base(), + overlay_accounts: HashMap::new(), + overlay_storage: HashMap::new(), + overlay_code_by_hash: HashMap::new(), storage_cleared: HashSet::new(), accounts_not_existing: HashSet::new(), block_hashes: HashMap::new(), - code_by_hash: HashMap::new(), block_number: Some(100), basefee: Some(1000), coinbase: None, diff --git a/tests/cow_snapshot.rs b/tests/cow_snapshot.rs index a89007b..9071cec 100644 --- a/tests/cow_snapshot.rs +++ b/tests/cow_snapshot.rs @@ -86,7 +86,11 @@ fn assert_equivalent(cache: &mut EvmCache, addrs: &[Address], slots: &[U256], la "{label}: block_number" ); assert_eq!(ov_cow.basefee(), ov_deep.basefee(), "{label}: basefee"); - assert_eq!(ov_cow.timestamp(), ov_deep.timestamp(), "{label}: timestamp"); + assert_eq!( + ov_cow.timestamp(), + ov_deep.timestamp(), + "{label}: timestamp" + ); for &a in addrs { let bc = ov_cow.basic(a).expect("cow basic"); @@ -158,15 +162,32 @@ async fn cow_snapshot_matches_deep_clone_through_mutations() -> Result<()> { // 4. write-through to an address PRESENT in layer 1 (shadowed there). cache.apply_updates(&[StateUpdate::slot(token, owner_bal, U256::from(2_000u64))]); - assert_equivalent(&mut cache, &addrs, &slots, "after write-through (in layer 1)"); + assert_equivalent( + &mut cache, + &addrs, + &slots, + "after write-through (in layer 1)", + ); // 5. write-through to an address ABSENT from layer 1 (layer-2-only — the §3 // footgun: the base must capture it). - cache.apply_updates(&[StateUpdate::slot(pool2, U256::from(7u64), U256::from(55u64))]); - assert_equivalent(&mut cache, &addrs, &slots, "after write-through (layer-2-only)"); + cache.apply_updates(&[StateUpdate::slot( + pool2, + U256::from(7u64), + U256::from(55u64), + )]); + assert_equivalent( + &mut cache, + &addrs, + &slots, + "after write-through (layer-2-only)", + ); // 6. relative native-balance delta. - cache.apply_updates(&[StateUpdate::balance_delta(owner, SlotDelta::Add(U256::from(500)))]); + cache.apply_updates(&[StateUpdate::balance_delta( + owner, + SlotDelta::Add(U256::from(500)), + )]); assert_equivalent(&mut cache, &addrs, &slots, "after balance delta"); // 7. committing revm call (mutates layer 1 only — never stales the base). @@ -178,7 +199,12 @@ async fn cow_snapshot_matches_deep_clone_through_mutations() -> Result<()> { cache.inject_storage_batch(&[(pool, U256::from(0u64), U256::from(111u64))]); assert_equivalent(&mut cache, &addrs, &slots, "after inject (new)"); cache.inject_storage_batch(&[(pool, U256::from(0u64), U256::from(222u64))]); - assert_equivalent(&mut cache, &addrs, &slots, "after inject (overwrite, same len)"); + assert_equivalent( + &mut cache, + &addrs, + &slots, + "after inject (overwrite, same len)", + ); // 9. simulated UNCONTROLLED layer-2 growth (a lazy RPC fetch / prefetch writes // `BlockchainDb` from inside foundry-fork-db, bypassing our write funnel): @@ -204,7 +230,12 @@ async fn cow_snapshot_matches_deep_clone_through_mutations() -> Result<()> { .or_default() .insert(U256::from(1u64), U256::from(333u64)); } - assert_equivalent(&mut cache, &addrs, &slots, "after uncontrolled layer-2 growth"); + assert_equivalent( + &mut cache, + &addrs, + &slots, + "after uncontrolled layer-2 growth", + ); // 10. purge. cache.purge_account(owner); @@ -234,6 +265,43 @@ async fn cow_snapshot_matches_deep_clone_through_mutations() -> Result<()> { Ok(()) } +/// Escape-hatch re-honest hook (adversarial-review finding). A direct, out-of-band +/// layer-2 write through `blockchain_db()` that overwrites an existing slot at an +/// unchanged slot count is the one mutation the count-based growth scan cannot see, +/// so the memoized base can go stale. `invalidate_snapshot_base()` must restore +/// read-equivalence with the deep-clone reference. +#[tokio::test(flavor = "multi_thread")] +async fn invalidate_snapshot_base_rehonest_after_escape_hatch_write() -> Result<()> { + let mut cache = setup_cache().await?; + let pool = Address::repeat_byte(0x77); // layer-2-only, non-shadowed + let slot = U256::from(0u64); + + cache.inject_storage_batch(&[(pool, slot, U256::from(111u64))]); + let _warm = cache.create_snapshot(); // memoize the base at 111 + + // Out-of-band overwrite at unchanged length (bypasses the write funnel). + { + let bdb = cache.blockchain_db(); + bdb.storage() + .write() + .entry(pool) + .or_default() + .insert(slot, U256::from(222u64)); + } + + // The documented re-honest hook must make the next snapshot reflect the write. + cache.invalidate_snapshot_base(); + let cow = cache.create_snapshot(); + let deep = cache.create_snapshot_deep_clone(); + assert_eq!( + cow.storage_value(pool, slot), + deep.storage_value(pool, slot), + "invalidate_snapshot_base must re-honest the base after an out-of-band write" + ); + assert_eq!(cow.storage_value(pool, slot), Some(U256::from(222u64))); + Ok(()) +} + /// COW must not alias: a snapshot taken earlier is unaffected by a later mutation /// of the same address (the memoized base is rebuilt copy-on-write, not mutated). #[tokio::test(flavor = "multi_thread")] From 776fe3e8dc768f954a05e926b837821f6e0f8227 Mon Sep 17 00:00:00 2001 From: Kai Aldag Date: Tue, 16 Jun 2026 17:55:41 +0100 Subject: [PATCH 22/26] Configurable EVM shared-memory pre-allocation (SharedMemoryCapacity) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-context EVM working-memory buffer was hardcoded to 64 KB in two places (EvmCache + EvmOverlay), tuned for a state-heavy upstream workload. Make it a first-class knob: - `SharedMemoryCapacity { Fixed(usize), Auto }`, default `Fixed(64_000)`, configured via `EvmCacheBuilder::shared_memory_capacity`. `Fixed` pins the size (general users running wide fan-outs of small sims can lower it to cut per-overlay memory); `Auto` sizes from the chain state loaded at build time (e.g. a bincode state file) — `loaded_slots * 16`, clamped to a 64 KB floor / 4 MiB ceiling. - Resolution happens in the new `with_cache_capacity` constructor (the builder's worker; `with_cache`/`new`/`from_backend` keep their signatures, defaulting to Fixed(64_000)). `Auto` reads the post-load layer-2 slot count, so it captures the maintain-list filter and any source, not just the raw file. - The resolved size is stored on EvmCache, exposed via `EvmCache::shared_memory_capacity()`, raised by `reserve_shared_memory`, and copied onto every EvmSnapshot so snapshot-backed EvmOverlays pre-allocate the same amount (overlay gains a `buffer_capacity` field; the hardcoded overlay constant is removed). Tests: a `resolve` heuristic unit test (floor/linear/ceiling, both feature configs) and `tests/shared_memory_capacity.rs` end-to-end over the builder (default, Fixed, Auto-with-no-state floor, and Auto sizing 10k loaded slots → 160_000). Full suite 335 (default) / 282 (--no-default-features); fmt + clippy (both configs) + doc + bench --no-run clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 10 ++ src/cache/mod.rs | 186 ++++++++++++++++++++++++++++++-- src/cache/overlay.rs | 27 +++-- src/cache/snapshot.rs | 7 ++ tests/shared_memory_capacity.rs | 114 ++++++++++++++++++++ 5 files changed, 326 insertions(+), 18 deletions(-) create mode 100644 tests/shared_memory_capacity.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 11dcc9e..7cb85c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -161,6 +161,16 @@ pre-release development phases (see [`docs/ROADMAP.md`](docs/ROADMAP.md)). exactly like a freshly-built overlay. The 64 KB shared-memory buffer is also recycled across the build→transact→revert call methods (stored as a plain `Vec`, so the overlay stays `Send`). +- **Configurable EVM shared-memory pre-allocation** — `SharedMemoryCapacity` + (`Fixed(usize)` / `Auto`, default `Fixed(64_000)`) set via + `EvmCacheBuilder::shared_memory_capacity`. `Fixed` pins the per-context working- + memory buffer (general users running wide fan-outs of small simulations can lower + it to cut per-overlay memory; the previous behavior is the default); `Auto` sizes + it from the chain state loaded at build time (e.g. a bincode state file), clamped + to a 64 kB floor / 4 MiB ceiling. The resolved size is readable via + `EvmCache::shared_memory_capacity()` and is propagated to every snapshot so + snapshot-backed overlays pre-allocate the same amount. `with_cache_capacity` is + the lower-level constructor behind the builder setter. ### Changed diff --git a/src/cache/mod.rs b/src/cache/mod.rs index 9a1e2d9..53ae869 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -300,6 +300,7 @@ pub struct EvmCacheBuilder

{ block: Option, cache_config: Option, spec_id: SpecId, + shared_memory_capacity: SharedMemoryCapacity, } impl

EvmCacheBuilder

@@ -313,6 +314,7 @@ where block: None, cache_config: None, spec_id: SpecId::CANCUN, + shared_memory_capacity: SharedMemoryCapacity::default(), } } @@ -354,9 +356,28 @@ where self } + /// Set how much EVM shared memory to pre-allocate per simulation context. + /// + /// Defaults to [`SharedMemoryCapacity::Fixed`]`(64_000)` (today's behavior). + /// Use `Fixed(n)` to pin a size, or [`SharedMemoryCapacity::Auto`] to size it + /// from the chain state loaded at [`build`](Self::build) time (e.g. a bincode + /// state file supplied via [`cache_config`](Self::cache_config)). See + /// [`SharedMemoryCapacity`] for the trade-offs. + pub fn shared_memory_capacity(mut self, capacity: SharedMemoryCapacity) -> Self { + self.shared_memory_capacity = capacity; + self + } + /// Build the [`EvmCache`], fetching the pinned block's header for context. pub async fn build(self) -> EvmCache { - EvmCache::with_cache(self.provider, self.block, self.cache_config, self.spec_id).await + EvmCache::with_cache_capacity( + self.provider, + self.block, + self.cache_config, + self.spec_id, + self.shared_memory_capacity, + ) + .await } } @@ -368,10 +389,67 @@ type InspectorCacheEvm<'a, INSP> = revm::MainnetEvm< INSP, >; -/// Default initial capacity for shared memory buffer. -/// Set to 64KB based on profiling (16x the REVM default of 4KB). -/// This eliminates reallocation during typical simulations with headroom. -const DEFAULT_SHARED_MEMORY_CAPACITY: usize = 64 * 1024; +/// Default initial capacity for the EVM shared-memory (working-memory) buffer. +/// 64 kB, chosen from profiling a state-heavy workload (16x the revm default of +/// 4 kB) so simulations rarely reallocate. Exposed for tuning via +/// [`SharedMemoryCapacity`]. +const DEFAULT_SHARED_MEMORY_CAPACITY: usize = 64_000; + +/// How much EVM shared memory (per-context working memory) to pre-allocate for +/// simulations. +/// +/// revm grows its shared memory on demand during execution; pre-allocating just +/// avoids repeated reallocations when simulations touch a lot of memory — the +/// original motivation was a state-heavy workload where resizing was hot. The +/// trade-off cuts both ways: a wide parallel fan-out of *small* simulations pays +/// this much memory per overlay, so general users may want a smaller `Fixed` size, +/// while state-heavy users can raise it or let it auto-size from the loaded state. +/// +/// The default is `Fixed(64_000)` (today's behavior). Configure it on +/// [`EvmCacheBuilder::shared_memory_capacity`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SharedMemoryCapacity { + /// Pre-allocate exactly this many bytes. The [`Default`] is `Fixed(64_000)`. + Fixed(usize), + /// Size the buffer from the amount of chain state loaded into the cache at + /// construction (e.g. from a bincode state file via + /// [`CacheConfig`]/[`EvmCacheBuilder::cache_config`]), clamped to a sane + /// floor/ceiling. Falls back to the floor when nothing is loaded. + /// + /// This is a heuristic proxy — persisted state size loosely correlates with the + /// working-set size of simulations over it, not an exact peak-memory model. Use + /// `Fixed` when you have profiled your workload. + Auto, +} + +impl Default for SharedMemoryCapacity { + fn default() -> Self { + Self::Fixed(DEFAULT_SHARED_MEMORY_CAPACITY) + } +} + +impl SharedMemoryCapacity { + /// Floor for [`Auto`](Self::Auto) (and the default fixed size): 64 kB. + pub const MIN_AUTO: usize = DEFAULT_SHARED_MEMORY_CAPACITY; + /// Ceiling for [`Auto`](Self::Auto): 4 MiB. A simulation that needs more than + /// this still works — revm grows the buffer past it on demand. + pub const MAX_AUTO: usize = 4 * 1024 * 1024; + /// Heuristic proxy: bytes of pre-allocated working memory per loaded storage + /// slot. Tune if profiling warrants. + const AUTO_BYTES_PER_SLOT: usize = 16; + + /// Resolve to a concrete byte capacity. `loaded_slots` is the number of layer-2 + /// storage slots present in the cache at construction (0 when nothing is + /// loaded); it is consulted only for [`Auto`](Self::Auto). + pub(crate) fn resolve(self, loaded_slots: usize) -> usize { + match self { + Self::Fixed(bytes) => bytes, + Self::Auto => loaded_slots + .saturating_mul(Self::AUTO_BYTES_PER_SLOT) + .clamp(Self::MIN_AUTO, Self::MAX_AUTO), + } + } +} /// EVM cache with lazy-loading RPC backend. /// @@ -452,6 +530,12 @@ pub struct EvmCache { /// uncontrolled lazy-fetch growth that bypasses the write funnel. Not /// serialized. base_storage_lens: HashMap, + /// Resolved per-context EVM shared-memory pre-allocation (bytes), from the + /// [`SharedMemoryCapacity`] at construction (resolving `Auto` against the loaded + /// state). Propagated to each [`EvmSnapshot`] so snapshot-backed overlays + /// pre-allocate the same amount. See + /// [`shared_memory_capacity`](Self::shared_memory_capacity). + shared_memory_capacity: usize, } /// Outcome of a balance-delta-tracking simulation. @@ -586,6 +670,31 @@ impl EvmCache { cache_config: Option, spec_id: SpecId, ) -> Self + where + P: Provider + 'static, + { + Self::with_cache_capacity( + provider, + block, + cache_config, + spec_id, + SharedMemoryCapacity::default(), + ) + .await + } + + /// Like [`with_cache`](Self::with_cache) but takes an explicit + /// [`SharedMemoryCapacity`] controlling per-context EVM working-memory + /// pre-allocation. This is what [`EvmCacheBuilder::build`] calls; prefer the + /// builder. With [`SharedMemoryCapacity::Auto`] the buffer is sized from the + /// layer-2 storage loaded at construction (e.g. a bincode state file). + pub async fn with_cache_capacity

( + provider: Arc

, + block: Option, + cache_config: Option, + spec_id: SpecId, + shared_memory_capacity: SharedMemoryCapacity, + ) -> Self where P: Provider + 'static, { @@ -904,6 +1013,20 @@ impl EvmCache { // Extract chain_id from cache config if available, default to Arbitrum let chain_id = cache_config.as_ref().map(|c| c.chain_id).unwrap_or(42161); + // Resolve the shared-memory pre-allocation. For `Auto` we size from the + // amount of layer-2 chain state actually loaded (post-filter), so a large + // bincode state file yields a larger buffer; `Fixed` ignores the count. + let loaded_slots = match shared_memory_capacity { + SharedMemoryCapacity::Auto => blockchain_db + .storage() + .read() + .values() + .map(|s| s.len()) + .sum(), + SharedMemoryCapacity::Fixed(_) => 0, + }; + let shared_memory_capacity = shared_memory_capacity.resolve(loaded_slots); + Self { backend, blockchain_db, @@ -921,9 +1044,7 @@ impl EvmCache { coinbase, prevrandao, block_gas_limit, - shared_memory_buffer: Rc::new(RefCell::new(Vec::with_capacity( - DEFAULT_SHARED_MEMORY_CAPACITY, - ))), + shared_memory_buffer: Rc::new(RefCell::new(Vec::with_capacity(shared_memory_capacity))), rpc_caller: Some(rpc_caller), storage_batch_fetcher: Some(storage_batch_fetcher), batch_block_id, @@ -933,6 +1054,7 @@ impl EvmCache { base_dirty: HashSet::new(), base_full_rebuild: false, base_storage_lens: HashMap::new(), + shared_memory_capacity, } } @@ -1011,6 +1133,7 @@ impl EvmCache { base_dirty: HashSet::new(), base_full_rebuild: false, base_storage_lens: HashMap::new(), + shared_memory_capacity: DEFAULT_SHARED_MEMORY_CAPACITY, } } @@ -2147,6 +2270,7 @@ impl EvmCache { chain_id: self.chain_id, timestamp: self.timestamp_override, spec_id: self.spec_id, + shared_memory_capacity: self.shared_memory_capacity, }) } @@ -2416,6 +2540,7 @@ impl EvmCache { chain_id: self.chain_id, timestamp: self.timestamp_override, spec_id: self.spec_id, + shared_memory_capacity: self.shared_memory_capacity, }) } @@ -3275,6 +3400,22 @@ impl EvmCache { "Reserved shared memory buffer capacity" ); } + drop(buffer); + // Record the high-water mark so snapshots taken afterwards propagate it to + // their overlays (snapshots copy the capacity at creation time). + self.shared_memory_capacity = self.shared_memory_capacity.max(capacity); + } + + /// The resolved per-context EVM shared-memory pre-allocation, in bytes. + /// + /// This is the [`SharedMemoryCapacity`] configured on the + /// [`EvmCacheBuilder`] resolved to a concrete size (with + /// [`SharedMemoryCapacity::Auto`] resolved against the state loaded at + /// construction), raised by any later [`reserve_shared_memory`](Self::reserve_shared_memory). + /// Each [`create_snapshot`](Self::create_snapshot) copies it onto the snapshot + /// so snapshot-backed [`EvmOverlay`]s pre-allocate the same amount. + pub fn shared_memory_capacity(&self) -> usize { + self.shared_memory_capacity } /// Purge all storage slots for a specific pool from both cache layers. @@ -4182,6 +4323,35 @@ fn extract_access_list(state: &revm::state::EvmState) -> AccessList { AccessList(items) } +#[cfg(test)] +mod shared_memory_capacity_tests { + use super::SharedMemoryCapacity as Cap; + + #[test] + fn default_is_fixed_64k() { + assert_eq!(Cap::default(), Cap::Fixed(64_000)); + } + + #[test] + fn fixed_ignores_loaded_slots() { + assert_eq!(Cap::Fixed(8_192).resolve(10_000_000), 8_192); + assert_eq!(Cap::Fixed(0).resolve(123), 0); + } + + #[test] + fn auto_floors_clamps_and_scales() { + // Nothing / little loaded → floor. + assert_eq!(Cap::Auto.resolve(0), Cap::MIN_AUTO); + assert_eq!(Cap::Auto.resolve(1_000), Cap::MIN_AUTO); // 16 KB < 64 KB floor + // Linear region (16 bytes/slot). + assert_eq!(Cap::Auto.resolve(10_000), 160_000); + assert_eq!(Cap::Auto.resolve(100_000), 1_600_000); + // Ceiling. + assert_eq!(Cap::Auto.resolve(usize::MAX), Cap::MAX_AUTO); + assert_eq!(Cap::Auto.resolve(262_144), Cap::MAX_AUTO); // 262_144 * 16 == 4 MiB + } +} + #[cfg(all(test, feature = "protocols"))] mod tests { use super::*; diff --git a/src/cache/overlay.rs b/src/cache/overlay.rs index 4e3fe38..2d84fe2 100644 --- a/src/cache/overlay.rs +++ b/src/cache/overlay.rs @@ -21,9 +21,6 @@ use crate::access_set::StorageAccessList; use crate::errors::{SimError, SimulationError, SimulationResult}; use crate::inspector::TransferInspector; -/// Default initial capacity for shared memory buffer (64KB). -const OVERLAY_SHARED_MEMORY_CAPACITY: usize = 64 * 1024; - type OverlayEvm<'a> = revm::MainnetEvm< Context, ()>, >; @@ -63,17 +60,28 @@ pub struct EvmOverlay { /// for revm's [`LocalContext`], runs, then reclaims and clears it after the /// EVM is dropped (see [`Self::build_evm_with_local`]). reusable_buffer: Vec, + /// Target pre-allocation (bytes) for [`Self::reusable_buffer`] and each + /// per-call buffer, taken from the snapshot's configured + /// [`SharedMemoryCapacity`](super::SharedMemoryCapacity) so overlays honor the + /// capacity set on the originating [`EvmCache`]. + buffer_capacity: usize, } impl EvmOverlay { /// Create a new overlay on the given snapshot. + /// + /// The reusable shared-memory buffer is pre-allocated to the snapshot's + /// configured shared-memory capacity (see + /// [`SharedMemoryCapacity`](super::SharedMemoryCapacity)). pub fn new(snapshot: Arc, ext_db: Option) -> Self { + let buffer_capacity = snapshot.shared_memory_capacity; Self { snapshot, dirty_accounts: HashMap::new(), dirty_storage: HashMap::new(), ext_db, - reusable_buffer: Vec::with_capacity(OVERLAY_SHARED_MEMORY_CAPACITY), + reusable_buffer: Vec::with_capacity(buffer_capacity), + buffer_capacity, } } @@ -133,11 +141,9 @@ impl EvmOverlay { /// Used by the public [`Self::build_evm`], which hands out the EVM and cannot /// reclaim its buffer afterwards. The internal call methods instead recycle /// [`Self::reusable_buffer`] via [`Self::build_evm_with_local`]. - fn fresh_local() -> LocalContext { + fn fresh_local(&self) -> LocalContext { LocalContext { - shared_memory_buffer: Rc::new(RefCell::new(Vec::with_capacity( - OVERLAY_SHARED_MEMORY_CAPACITY, - ))), + shared_memory_buffer: Rc::new(RefCell::new(Vec::with_capacity(self.buffer_capacity))), precompile_error_message: None, } } @@ -212,7 +218,7 @@ impl EvmOverlay { /// Note: The returned EVM is `!Send` (due to `LocalContext`'s `Rc`), /// but this is fine because it's created and used within a single task. pub fn build_evm(&mut self) -> OverlayEvm<'_> { - let local = Self::fresh_local(); + let local = self.fresh_local(); self.build_evm_with_local(local) } @@ -300,7 +306,7 @@ impl EvmOverlay { buf.clear(); self.reusable_buffer = buf; } else { - self.reusable_buffer = Vec::with_capacity(OVERLAY_SHARED_MEMORY_CAPACITY); + self.reusable_buffer = Vec::with_capacity(self.buffer_capacity); } } @@ -805,6 +811,7 @@ mod tests { chain_id: 42161, timestamp: None, spec_id: SpecId::CANCUN, + shared_memory_capacity: 64_000, }) } diff --git a/src/cache/snapshot.rs b/src/cache/snapshot.rs index 3e3e50f..0b6c5ec 100644 --- a/src/cache/snapshot.rs +++ b/src/cache/snapshot.rs @@ -112,6 +112,12 @@ pub struct EvmSnapshot { pub(crate) chain_id: u64, pub(crate) timestamp: Option, pub(crate) spec_id: SpecId, + /// Per-context EVM shared-memory pre-allocation (bytes) copied from the + /// [`EvmCache`](super::EvmCache) at snapshot time, so an [`EvmOverlay`] built + /// from this snapshot pre-allocates the same working-memory size the live cache + /// was configured with (see + /// [`SharedMemoryCapacity`](super::SharedMemoryCapacity)). + pub(crate) shared_memory_capacity: usize, } impl EvmSnapshot { @@ -202,6 +208,7 @@ mod tests { chain_id: 42161, timestamp: None, spec_id: SpecId::CANCUN, + shared_memory_capacity: 64_000, }; assert_eq!(snap.chain_id, 42161); assert_eq!(snap.block_number, Some(100)); diff --git a/tests/shared_memory_capacity.rs b/tests/shared_memory_capacity.rs new file mode 100644 index 0000000..08a9bf4 --- /dev/null +++ b/tests/shared_memory_capacity.rs @@ -0,0 +1,114 @@ +//! Offline tests for the configurable EVM shared-memory pre-allocation +//! ([`SharedMemoryCapacity`]) wired through [`EvmCacheBuilder`]. +//! +//! Covers the three user-facing behaviors: the default, an explicit `Fixed` size, +//! and `Auto` sizing from the chain state loaded at build time (the +//! "intelligently allocate from a bincode state file" path). All offline. + +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +use alloy_primitives::{Address, U256}; +use alloy_provider::RootProvider; +use alloy_provider::network::AnyNetwork; +use alloy_rpc_client::RpcClient; +use alloy_transport::mock::Asserter; +use anyhow::Result; +use evm_fork_cache::cache::{CacheConfig, EvmCacheBuilder, SharedMemoryCapacity}; + +fn mock_provider() -> Arc> { + Arc::new(RootProvider::::new(RpcClient::mocked( + Asserter::new(), + ))) +} + +/// A unique temp dir for a disk-backed cache (no two tests collide). +fn unique_cache_dir(tag: &str) -> std::path::PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + std::env::temp_dir().join(format!("evm_fork_cache_smc_{tag}_{nanos}")) +} + +#[tokio::test(flavor = "multi_thread")] +async fn default_capacity_is_fixed_64k() -> Result<()> { + let cache = EvmCacheBuilder::new(mock_provider()).build().await; + assert_eq!( + cache.shared_memory_capacity(), + 64_000, + "the default must be Fixed(64_000)" + ); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn fixed_capacity_is_honored() -> Result<()> { + let cache = EvmCacheBuilder::new(mock_provider()) + .shared_memory_capacity(SharedMemoryCapacity::Fixed(8_192)) + .build() + .await; + assert_eq!(cache.shared_memory_capacity(), 8_192); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn auto_capacity_with_no_loaded_state_falls_back_to_floor() -> Result<()> { + // No cache_config → nothing loaded → Auto resolves to the 64 KB floor. + let cache = EvmCacheBuilder::new(mock_provider()) + .shared_memory_capacity(SharedMemoryCapacity::Auto) + .build() + .await; + assert_eq!( + cache.shared_memory_capacity(), + SharedMemoryCapacity::MIN_AUTO + ); + Ok(()) +} + +/// The headline: `Auto` sizes the buffer from the chain state in a loaded bincode +/// state file. A first cache persists 10 000 storage slots; a second cache built +/// with `Auto` over the same `CacheConfig` loads them and pre-allocates +/// `10_000 * 16 = 160_000` bytes (vs. the 64 KB default). +#[tokio::test(flavor = "multi_thread")] +async fn auto_capacity_scales_with_loaded_binary_state() -> Result<()> { + let dir = unique_cache_dir("auto"); + let cfg = CacheConfig::new(&dir, 1, Default::default(), Default::default()); + + // First cache: seed 10k slots into layer 2 and persist to the bincode state file. + { + let mut cache = EvmCacheBuilder::new(mock_provider()) + .cache_config(cfg.clone()) + .build() + .await; + let token = Address::repeat_byte(0x11); + let batch: Vec<(Address, U256, U256)> = (0..10_000u64) + .map(|i| (token, U256::from(i), U256::from(i + 1))) + .collect(); + cache.inject_storage_batch(&batch); + cache.flush(); // writes evm_state.bin + } + + // Second cache: Auto over the same config loads the 10k slots and sizes from them. + let reloaded = EvmCacheBuilder::new(mock_provider()) + .cache_config(cfg.clone()) + .shared_memory_capacity(SharedMemoryCapacity::Auto) + .build() + .await; + assert_eq!( + reloaded.shared_memory_capacity(), + 160_000, + "Auto must size from the 10k loaded slots (10_000 * 16 bytes)" + ); + + // A Fixed override ignores the loaded state. + let fixed = EvmCacheBuilder::new(mock_provider()) + .cache_config(cfg.clone()) + .shared_memory_capacity(SharedMemoryCapacity::Fixed(64_000)) + .build() + .await; + assert_eq!(fixed.shared_memory_capacity(), 64_000); + + let _ = std::fs::remove_dir_all(&dir); + Ok(()) +} From eb0bad6d7057be61bf81a3b410a09783f5016362 Mon Sep 17 00:00:00 2001 From: Kai Aldag Date: Wed, 17 Jun 2026 09:40:57 +0100 Subject: [PATCH 23/26] Deflake the drop-abort freshness test with a deterministic gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `dropping_speculative_sim_aborts_before_queueing_correction` assumed the SpeculativeSim's drop-abort would win a race against the spawned multi-thread validator's first poll, which fails intermittently under full-suite parallel load. Replace the racy "called" atomic flag with a `Gate` (Mutex + Condvar): the fetcher blocks until the test releases the gate, and the test releases it only *after* `drop(sim)` sets the cancel flag. So the validator's fetch — and thus its post-fetch, correction-queuing checkpoint — can only complete once cancellation is already observable, regardless of scheduler interleaving. Drops the over-strict "fetcher never reached" assertion (the product guarantees a cancel seen at a checkpoint suppresses side effects, not that an in-flight fetch is skipped) and keeps the real invariants: no correction queued, no re-run. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/common/mod.rs | 63 ++++++++++++++++++++++++++++++++++++++------- tests/freshness.rs | 38 +++++++++++++++++---------- 2 files changed, 78 insertions(+), 23 deletions(-) diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 509a1f5..196f3d3 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -6,7 +6,7 @@ #![allow(dead_code)] use std::collections::HashMap; -use std::sync::Arc; +use std::sync::{Arc, Condvar, Mutex}; use alloy_eips::BlockId; use alloy_primitives::{Address, Bytes, U256, hex}; @@ -128,19 +128,64 @@ pub fn failing_fetcher() -> StorageBatchFetchFn { }) } -/// Build a stub [`StorageBatchFetchFn`] that reports chosen values *and* flips a -/// shared flag the first time it is called. +/// A one-shot synchronous gate: a holder blocks in [`wait`](Gate::wait) until +/// some other thread calls [`release`](Gate::release). Cloning shares the same +/// underlying state, and `release` is sticky — once released, every present and +/// future `wait` returns immediately. /// -/// Used by the Drop-abort test to prove the background validator was cancelled -/// before it ever fetched (so it could not have queued a correction). The -/// returned values otherwise behave exactly like [`stub_fetcher`]. -pub fn tracking_fetcher( +/// Used by the Drop-abort test to make the background validator's fetch +/// deterministically ordered *after* the drop. The fetcher (running on a worker +/// thread) cannot return — and therefore the validator cannot reach its +/// post-fetch checkpoint — until the test has dropped the `SpeculativeSim` and +/// released the gate, eliminating the spawn/poll race regardless of how the +/// multi-thread scheduler interleaves the two threads. +/// +/// Built on a `Mutex` + `Condvar` so the whole thing is `Send + Sync`, +/// which a [`StorageBatchFetchFn`] closure must be. +#[derive(Clone, Default)] +pub struct Gate { + inner: Arc<(Mutex, Condvar)>, +} + +impl Gate { + pub fn new() -> Self { + Self::default() + } + + /// Block until [`release`](Gate::release) has been called (returns + /// immediately if it already has). + pub fn wait(&self) { + let (lock, cv) = &*self.inner; + let mut released = lock.lock().unwrap_or_else(|e| e.into_inner()); + while !*released { + released = cv.wait(released).unwrap_or_else(|e| e.into_inner()); + } + } + + /// Wake any current waiter and let all future waiters pass. + pub fn release(&self) { + let (lock, cv) = &*self.inner; + *lock.lock().unwrap_or_else(|e| e.into_inner()) = true; + cv.notify_all(); + } +} + +/// Build a stub [`StorageBatchFetchFn`] that reports chosen values but blocks on +/// `gate` before returning. +/// +/// Used by the Drop-abort test: by releasing the gate only *after* dropping the +/// `SpeculativeSim`, the test guarantees the validator's fetch completes (and so +/// its post-fetch, correction-queuing checkpoint runs) strictly after the +/// cancel flag is set — so the dropped speculation can never queue a correction, +/// no matter how the scheduler races the two threads. The returned values +/// otherwise behave exactly like [`stub_fetcher`]. +pub fn gated_tracking_fetcher( values: HashMap<(Address, U256), U256>, - called: Arc, + gate: Gate, ) -> StorageBatchFetchFn { Arc::new( move |requests: Vec<(Address, U256)>, _block: Option| { - called.store(true, std::sync::atomic::Ordering::SeqCst); + gate.wait(); requests .into_iter() .map(|(addr, slot)| { diff --git a/tests/freshness.rs b/tests/freshness.rs index 353f827..73607d0 100644 --- a/tests/freshness.rs +++ b/tests/freshness.rs @@ -16,8 +16,8 @@ use alloy_sol_types::SolCall; use anyhow::Result; use common::{ - MOCK_ERC20_BALANCE_SLOT, MockERC20, failing_fetcher, install_default_account, - install_mock_erc20, panicking_fetcher, setup_cache, stub_fetcher, tracking_fetcher, + Gate, MOCK_ERC20_BALANCE_SLOT, MockERC20, failing_fetcher, gated_tracking_fetcher, + install_default_account, install_mock_erc20, panicking_fetcher, setup_cache, stub_fetcher, }; use evm_fork_cache::cache::{ EvmCache, EvmOverlay, SimStatus, SlotObservationTracker, StorageBatchFetchFn, @@ -1224,9 +1224,20 @@ async fn run_into_optimistic_aborts_validation() -> Result<()> { // T3 (part 1): dropping the SpeculativeSim (no validate/into_optimistic) aborts // the validation task before it can push a correction. The fetcher reports a -// CHANGED value and flips a "called" flag; after the drop + settle we assert the -// pending queue is empty (and, robustly, that the fetcher was never even -// reached) — proving the abort beat the push. +// CHANGED value, so an *uncancelled* validator would queue a correction and bump +// the re-run count; we assert neither happens after the drop. +// +// Determinism: the validator's only correction-queuing path runs *after* its +// fetch returns (the post-fetch cancel checkpoint in `run_validator` gates it). +// We make that ordering race-free with a gate the test controls — the fetcher +// blocks until `gate.release()`, and we release only *after* `drop(sim)` has set +// the cancel flag. So however the multi-thread scheduler interleaves the spawned +// task and this thread, the fetch (and thus the post-fetch checkpoint) can only +// complete once cancellation is already observable, and the correction is +// suppressed. We deliberately do NOT assert the fetcher was never reached: the +// product only guarantees a cancel seen at a checkpoint suppresses side effects, +// not that an in-flight fetch is skipped — asserting the latter was the original +// over-strict, racy condition. #[tokio::test(flavor = "multi_thread")] async fn dropping_speculative_sim_aborts_before_queueing_correction() -> Result<()> { let token = Address::repeat_byte(0x44); @@ -1234,10 +1245,10 @@ async fn dropping_speculative_sim_aborts_before_queueing_correction() -> Result< let recipient = Address::repeat_byte(0x66); let mut cache = cache_with_balance(token, owner, U256::from(1000)).await?; - let called = Arc::new(std::sync::atomic::AtomicBool::new(false)); - cache.set_storage_batch_fetcher(tracking_fetcher( + let gate = Gate::new(); + cache.set_storage_batch_fetcher(gated_tracking_fetcher( HashMap::from([((token, balance_slot_for(owner)), U256::from(50))]), - Arc::clone(&called), + gate.clone(), )); let mut controller = FreshnessController::new(FreshnessRegistry::new(), AlwaysVerify); @@ -1249,9 +1260,12 @@ async fn dropping_speculative_sim_aborts_before_queueing_correction() -> Result< transfer_calldata(recipient, U256::from(100)), )], )?; - // Drop immediately, with NO intervening await, so the abort flag is set - // before the spawned task is ever polled. + // Drop with NO intervening await, then release the gate. Releasing only after + // the drop guarantees the validator's fetch (if it even reaches it) returns + // strictly after the cancel flag is set, so its post-fetch checkpoint bails + // out before queuing anything. drop(sim); + gate.release(); settle().await; @@ -1260,10 +1274,6 @@ async fn dropping_speculative_sim_aborts_before_queueing_correction() -> Result< 0, "dropping the sim must abort validation before it queues a correction" ); - assert!( - !called.load(std::sync::atomic::Ordering::SeqCst), - "the aborted validator should never have reached the fetcher" - ); assert_eq!(controller.rerun_count(), 0, "no re-run after abort"); Ok(()) } From 85c3cf84cd39b3a7f9c924eb4e394582f2c4e702 Mon Sep 17 00:00:00 2001 From: Kai Aldag Date: Wed, 17 Jun 2026 09:57:32 +0100 Subject: [PATCH 24/26] Phase 5 review fixes: prune stale COW code_by_hash + doc/test gaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses three PR-review findings on the copy-on-write snapshot: - P2 (correctness): refresh_base's Case-4 partial rebuild cloned the previous code_by_hash and only added refreshed dirty-account codes, so a purged or recoded account left a stale hash. A direct EvmOverlay::code_by_hash(old_hash) then returned removed bytecode while create_snapshot_deep_clone (which rebuilds the index from current accounts) returned none — a read-equivalence violation and a slow memory leak. Fix: rebuild the index from the refreshed accounts via a shared `code_index` helper used by both build_base_full and the Case-4 path, so the two stay in lockstep; handles shared hashes (a hash survives iff some present account still carries it) and prunes unreferenced ones. - P3 (coverage): the differential gate now also compares code_by_hash for each probed account's code hash, and a new regression test (cow_code_index_matches_deep_clone_after_base_account_recoded) warms the base with bytecode, recodes the account, dirties it via a controlled per-address write (Case-4 partial rebuild), and asserts the old hash no longer resolves. Verified red against the pre-fix code. - P3 (docs): create_snapshot rustdoc no longer claims it "merges both layers into a single flat HashMap"; it now describes the memoized layer-2 base + layer-1 overlay fold and the &mut self receiver. Tests 329 (default) / 276 (--no-default-features); fmt + clippy (both configs) + doc clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/cache/mod.rs | 65 +++++++++++++++++++++---------- tests/cow_snapshot.rs | 91 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 133 insertions(+), 23 deletions(-) diff --git a/src/cache/mod.rs b/src/cache/mod.rs index 9a1e2d9..0d39809 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -2067,17 +2067,21 @@ impl EvmCache { } } - /// Create an immutable snapshot of the current EVM state for cross-thread - /// fan-out. - /// - /// Merges both layers (CacheDB overlay + BlockchainDb backend) into a - /// single flat HashMap. The snapshot is `Send + Sync` and can be shared - /// across threads via `Arc`. - /// - /// CacheDB overlay values take precedence over BlockchainDb values. - /// Use with [`EvmOverlay`] for parallel simulation. - /// - /// For cheap same-thread save/restore of just the overlay, prefer + /// Create an immutable, `Send + Sync` snapshot of the current EVM state for + /// cross-thread fan-out (the copy-on-write two-tier view, Pillar A). + /// + /// Rather than deep-copying both layers, this memoizes the cold layer-2 + /// (`BlockchainDb`) index as an `Arc`-shared base — reused as a cheap + /// `Arc::clone` when layer 2 is unchanged, rebuilt copy-on-write only for the + /// addresses that changed — and folds the hot layer-1 (`CacheDB` overlay) + /// delta over it. Layer-1 values shadow the base on reads, reproducing the + /// live cache's layered semantics; the resulting [`EvmSnapshot`] is shared + /// across threads via `Arc`. Its cost tracks *changed* state, not *total* + /// state. (The retained [`create_snapshot_deep_clone`](Self::create_snapshot_deep_clone) + /// is the read-equivalent O(total) reference, kept for benchmarking/testing.) + /// + /// Takes `&mut self` because it refreshes and memoizes the base. For cheap + /// same-thread save/restore of just the overlay, prefer /// [`snapshot`](Self::snapshot) / [`restore`](Self::restore) instead. pub fn create_snapshot(&mut self) -> Arc { // 1. Refresh / memoize the cold layer-2 base, then take a cheap Arc handle @@ -2238,18 +2242,14 @@ impl EvmCache { let prev = self.base.as_ref().expect("base present in case 4"); let mut accounts = prev.accounts.clone(); let mut storage = prev.storage.clone(); - let mut code_by_hash = prev.code_by_hash.clone(); let db_accounts = self.blockchain_db.accounts().read(); let db_storage = self.blockchain_db.storage().read(); for addr in self.base_dirty.iter().copied() { - // Account info + code: refresh from the current layer-2 account, or drop - // it if the account no longer exists in layer 2 (e.g. after a purge). + // Account info: refresh from the current layer-2 account, or drop it if + // the account no longer exists in layer 2 (e.g. after a purge). match db_accounts.get(&addr) { Some(info) => { - if let Some(code) = &info.code { - code_by_hash.insert(info.code_hash, code.clone()); - } accounts.insert(addr, info.clone()); } None => { @@ -2272,6 +2272,16 @@ impl EvmCache { } } } + drop(db_accounts); + drop(db_storage); + + // Rebuild the code index from the refreshed accounts (NOT cloned from the + // previous base): a purged or recoded dirty account must not leave a stale + // `code_by_hash` entry, which would diverge from `create_snapshot_deep_clone` + // on a direct `code_by_hash(old_hash)` lookup. Rebuilding from scratch also + // handles shared code hashes correctly (a hash survives iff some present + // account still carries it). + let code_by_hash = Self::code_index(&accounts); self.base = Some(Arc::new(snapshot::BaseState { accounts, @@ -2281,21 +2291,34 @@ impl EvmCache { self.base_dirty.clear(); } + /// Build the bytecode-by-hash index from a set of (layer-2) accounts, matching + /// the deep-clone reference: a hash is present iff some account carries that + /// code inline. Rebuilt from scratch on every base (re)build so a purged or + /// recoded account never leaves a stale entry — preserving read-equivalence + /// with [`create_snapshot_deep_clone`](Self::create_snapshot_deep_clone). + fn code_index(accounts: &HashMap) -> HashMap { + accounts + .values() + .filter_map(|info| { + info.code + .as_ref() + .map(|code| (info.code_hash, code.clone())) + }) + .collect() + } + /// Build a fresh [`BaseState`](snapshot::BaseState) by flattening all of layer /// 2, recording `base_storage_lens`. Shared by `refresh_base`'s full-rebuild /// path and [`create_snapshot_deep_clone`](Self::create_snapshot_deep_clone). fn build_base_full(&mut self) -> snapshot::BaseState { let mut accounts = HashMap::new(); - let mut code_by_hash = HashMap::new(); { let db_accounts = self.blockchain_db.accounts().read(); for (addr, info) in db_accounts.iter() { - if let Some(code) = &info.code { - code_by_hash.insert(info.code_hash, code.clone()); - } accounts.insert(*addr, info.clone()); } } + let code_by_hash = Self::code_index(&accounts); let mut storage = HashMap::new(); self.base_storage_lens.clear(); { diff --git a/tests/cow_snapshot.rs b/tests/cow_snapshot.rs index 9071cec..697e7f1 100644 --- a/tests/cow_snapshot.rs +++ b/tests/cow_snapshot.rs @@ -23,12 +23,12 @@ mod common; use std::sync::Arc; -use alloy_primitives::{Address, U256, keccak256}; +use alloy_primitives::{Address, Bytes, U256, keccak256}; use alloy_sol_types::{SolCall, SolValue}; use anyhow::{Result, anyhow}; use revm::database::AccountState; use revm::database_interface::Database; -use revm::state::AccountInfo; +use revm::state::{AccountInfo, Bytecode}; use common::{ MOCK_ERC20_BALANCE_SLOT, MockERC20, install_default_account, install_mock_erc20, setup_cache, @@ -99,6 +99,15 @@ fn assert_equivalent(cache: &mut EvmCache, addrs: &[Address], slots: &[U256], la account_eq(&bc, &bd), "{label}: basic mismatch at {a}: cow={bc:?} deep={bd:?}" ); + // Code lookup for the account's code hash must agree (spec §8.1). + if let Some(info) = &bc { + let h = info.code_hash; + assert_eq!( + ov_cow.code_by_hash(h).expect("cow code").original_bytes(), + ov_deep.code_by_hash(h).expect("deep code").original_bytes(), + "{label}: code_by_hash mismatch at {a} (hash {h})" + ); + } for &s in slots { assert_eq!( cow.storage_value(a, s), @@ -302,6 +311,84 @@ async fn invalidate_snapshot_base_rehonest_after_escape_hatch_write() -> Result< Ok(()) } +/// Regression (review finding P2): the COW partial rebuild must not leave a stale +/// `code_by_hash` entry when a base account is recoded or purged. Warm the base +/// with a code-bearing account, recode it in layer 2, dirty it via a controlled +/// per-address write (so `refresh_base` takes the Case-4 *partial* rebuild path, +/// not a full rebuild), and assert the old hash no longer resolves — matching the +/// deep-clone reference, which rebuilds its code index from current accounts. +#[tokio::test(flavor = "multi_thread")] +async fn cow_code_index_matches_deep_clone_after_base_account_recoded() -> Result<()> { + let mut cache = setup_cache().await?; + let contract = Address::repeat_byte(0xc0); + let code_v1 = Bytecode::new_raw(Bytes::from(vec![0x60u8, 0x01])); + let code_v2 = Bytecode::new_raw(Bytes::from(vec![0x60u8, 0x02, 0x60, 0x03])); + let h1 = code_v1.hash_slow(); + let h2 = code_v2.hash_slow(); + assert_ne!(h1, h2); + + // Seed a code-bearing account (code_v1) directly into the cold base (layer 2), + // then re-honest the memoized base. + let put_account = |cache: &EvmCache, code: &Bytecode, hash| { + cache.blockchain_db().accounts().write().insert( + contract, + AccountInfo { + balance: U256::from(1u64), + nonce: 1, + code_hash: hash, + code: Some(code.clone()), + account_id: None, + }, + ); + }; + put_account(&cache, &code_v1, h1); + cache.invalidate_snapshot_base(); + let warm = cache.create_snapshot(); // base now indexes h1 -> code_v1 + let mut ov_warm = EvmOverlay::new(Arc::clone(&warm), None); + assert_eq!( + ov_warm + .code_by_hash(h1) + .expect("warm code") + .original_bytes(), + code_v1.original_bytes(), + "warm snapshot must resolve the seeded code" + ); + + // Recode the base account to code_v2 (out-of-band), then dirty `contract` via a + // controlled per-address write so the next snapshot takes the Case-4 partial + // rebuild — exactly the path that previously failed to prune the old hash. + put_account(&cache, &code_v2, h2); + cache.apply_updates(&[StateUpdate::slot( + contract, + U256::from(0u64), + U256::from(9u64), + )]); + + let cow = cache.create_snapshot(); + let deep = cache.create_snapshot_deep_clone(); + let mut ov_cow = EvmOverlay::new(Arc::clone(&cow), None); + let mut ov_deep = EvmOverlay::new(Arc::clone(&deep), None); + + // The new hash resolves identically... + assert_eq!( + ov_cow.code_by_hash(h2).expect("cow h2").original_bytes(), + ov_deep.code_by_hash(h2).expect("deep h2").original_bytes(), + "new code hash must match the deep clone" + ); + // ...and the now-unreferenced old hash must NOT linger in the COW base: both + // resolve to empty (the deep clone never had it after the recode). + assert!( + ov_deep.code_by_hash(h1).expect("deep h1").is_empty(), + "sanity: deep clone drops the unreferenced old hash" + ); + assert_eq!( + ov_cow.code_by_hash(h1).expect("cow h1").original_bytes(), + ov_deep.code_by_hash(h1).expect("deep h1").original_bytes(), + "COW must not return stale bytecode for the recoded account's old hash" + ); + Ok(()) +} + /// COW must not alias: a snapshot taken earlier is unaffected by a later mutation /// of the same address (the memoized base is rebuilt copy-on-write, not mutated). #[tokio::test(flavor = "multi_thread")] From 6e317c5a1a63f0b8a8f99cfb50cb0b91a9491a44 Mon Sep 17 00:00:00 2001 From: Kai Aldag Date: Wed, 17 Jun 2026 12:50:19 +0100 Subject: [PATCH 25/26] address known issues --- CHANGELOG.md | 57 +++- Cargo.lock | 1 + Cargo.toml | 1 + README.md | 8 +- benches/simulation.rs | 21 +- docs/KNOWN_ISSUES.md | 198 +++++++------ docs/phase-2-spec.md | 2 +- docs/phase-3-spec.md | 40 +-- examples/prefetch_registry.rs | 5 +- src/access_list.rs | 279 +++++++++--------- src/cache/binary_state.rs | 136 ++++++--- src/cache/bytecode.rs | 65 ++++- src/cache/metadata.rs | 32 ++- src/cache/mod.rs | 468 ++++++++++++++++++++++--------- src/cache/tick_snapshot.rs | 28 +- src/cache/versioned.rs | 66 +++++ src/lib.rs | 4 +- src/prefetch_registry.rs | 72 +++-- src/state_update.rs | 150 ++++++---- tests/cache_state.rs | 77 ++++- tests/cow_snapshot.rs | 197 ++++++++++++- tests/event_pipeline.rs | 4 +- tests/freshness.rs | 6 +- tests/serialization_roundtrip.rs | 52 ++++ tests/shared_memory_capacity.rs | 37 ++- tests/state_update.rs | 65 ++++- 26 files changed, 1456 insertions(+), 615 deletions(-) create mode 100644 src/cache/versioned.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cb85c5..6808c8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -158,19 +158,33 @@ pre-release development phases (see [`docs/ROADMAP.md`](docs/ROADMAP.md)). many simulations against the same snapshot without reallocating: it clears the per-simulation dirty layer (keeping the snapshot `Arc`, `ext_db`, and the reusable shared-memory buffer), reading the pristine snapshot again and behaving - exactly like a freshly-built overlay. The 64 KB shared-memory buffer is also + exactly like a freshly-built overlay. The 64 KiB shared-memory buffer is also recycled across the build→transact→revert call methods (stored as a plain `Vec`, so the overlay stays `Send`). - **Configurable EVM shared-memory pre-allocation** — `SharedMemoryCapacity` - (`Fixed(usize)` / `Auto`, default `Fixed(64_000)`) set via + (`Fixed(usize)` / `Auto`, default `Fixed(64 * 1024)` / 65,536 bytes) set via `EvmCacheBuilder::shared_memory_capacity`. `Fixed` pins the per-context working- memory buffer (general users running wide fan-outs of small simulations can lower it to cut per-overlay memory; the previous behavior is the default); `Auto` sizes it from the chain state loaded at build time (e.g. a bincode state file), clamped - to a 64 kB floor / 4 MiB ceiling. The resolved size is readable via + to a 64 KiB floor / 4 MiB ceiling. The resolved size is readable via `EvmCache::shared_memory_capacity()` and is propagated to every snapshot so snapshot-backed overlays pre-allocate the same amount. `with_cache_capacity` is the lower-level constructor behind the builder setter. +- **Explicit cold-account materialization** — `StateUpdate::AccountUpsert` and + `StateUpdate::account_upsert(...)` intentionally materialize an account absent + from both layers. Normal `StateUpdate::Account` patches are now cold-aware and + surface skipped cold patches through `StateDiff.skipped_accounts: + Vec`. +- **Invalidating layer-2 mutation wrapper** — `EvmCache::with_blockchain_db_mut` + runs a synchronous direct `BlockchainDb` mutation and invalidates the Phase 5 + memoized COW base automatically after the closure returns. +- **Exact access-list RLP data-gas helper** — + `access_list::access_list_rlp_data_gas(&AccessList)` returns the EIP-2930 RLP + calldata gas for an access list and backs the L2 profitability calculation. +- **Versioned on-disk cache envelope** — binary EVM state, bytecode, + `ImmutableDataCache`, and V3 tick snapshot cache files now start with + crate-specific magic bytes plus a `u32` version before the bincode payload. ### Changed @@ -183,7 +197,25 @@ pre-release development phases (see [`docs/ROADMAP.md`](docs/ROADMAP.md)). layer-2 bulk write now marks the touched addresses dirty for the memoized copy-on-write base. The write itself is still a direct backend (layer-2) write with the same semantics; only the receiver mutability changed. - +- **Raw layer-2 handles were renamed to unchecked accessors** (Phase 5) — + `EvmCache::blockchain_db()` is now `unchecked_blockchain_db()` and + `EvmCache::backend()` is now `unchecked_backend()`. The rename makes the + bypass explicit; use `with_blockchain_db_mut` for synchronous direct writes that + should automatically invalidate the snapshot base. +- **Persistence APIs now return `Result<()>`** — `cache::save_binary_state`, + `PrefetchRegistry::save`, and `EvmCache::flush` report serialization, + directory-creation, and write failures to explicit callers. `Drop` remains + best-effort and logs `flush()` errors. +- **Block re-pins clear stale context** — `set_block` sets `block_number` only + for concrete numeric pins, clears it for tag/hash/`None` pins, and clears stale + `basefee` on block changes and on non-concrete pin calls that can drift under + the same tag. `repin_to_block` follows the same no-stale-basefee rule; callers + refresh `NUMBER`/`BASEFEE` via `set_block_context` after fetching the new + header. +- **Legacy raw-bincode cache files are treated as misses** — the versioned cache + envelope intentionally rejects unversioned `evm_state.bin`, `bytecodes.bin`, + `immutable_data.bin`, and `v3_tick_snapshots.bin` payloads rather than trying + to deserialize ambiguous layouts. - Simulation entry points that distinguish failure modes return `SimulationResult` (`Result`), separating decoded reverts, EVM halts, and host errors. `SimulationErrorKind` remains as a deprecated alias. @@ -199,6 +231,19 @@ pre-release development phases (see [`docs/ROADMAP.md`](docs/ROADMAP.md)). ### Fixed +- **Cold absolute account patches no longer mask on-chain accounts.** + `StateUpdate::Account` on an account absent from both layers now skips instead + of writing `AccountInfo::default()` fields through the shared backend. The + skipped patch is visible in `StateDiff.skipped_accounts`; intentional cold + creation uses `StateUpdate::AccountUpsert`. +- **Access-list profitability no longer conflates provider failures with + unprofitable lists.** `SmartAccessList::into_access_list_if_profitable` and + `access_list_if_profitable` now propagate provider/pricing failures as `Err` + and reserve `Ok(None)` for empty, zero-priced, or genuinely unprofitable lists. +- **`simulate_call_with_balance_deltas` now reports a real access list.** It + extracts the EIP-2930 touched account/slot list from the EVM journal before + commit/revert, including the pre/post `balanceOf` reads and the simulated call, + instead of returning `AccessList::default()`. - **`cached_storage_value` silent-corruption bug** (Phase 3 §16.0, audit HIGH + MED). For a storage slot absent from an overlay account whose revm `account_state` is `StorageCleared` or `NotExisting`, the accessor now returns @@ -214,7 +259,9 @@ pre-release development phases (see [`docs/ROADMAP.md`](docs/ROADMAP.md)). **skips both layer writes** (returning an empty diff) when no field actually changes, instead of unconditionally inserting `AccountInfo::default()` into the shared backend for an all-`None` (or value-unchanged) patch on an absent address. - A real field change still materializes the backend account (unchanged intent). + Phase 5 later tightened this further: real field changes on cold accounts now + skip through `StateDiff.skipped_accounts` unless the caller uses + `StateUpdate::AccountUpsert`. - **`account_state`-awareness extended to the snapshot + account-info paths** (Phase 3 fix-review, HIGH + MED). A follow-up adversarial review found the §16.0 `cached_storage_value` fix had not been propagated to two sibling read paths: diff --git a/Cargo.lock b/Cargo.lock index db4da26..dc8eaf5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1852,6 +1852,7 @@ dependencies = [ "alloy-node-bindings", "alloy-primitives", "alloy-provider", + "alloy-rlp", "alloy-rpc-client", "alloy-rpc-types-eth", "alloy-sol-types", diff --git a/Cargo.toml b/Cargo.toml index 4a928d4..5f7cf9b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ alloy-eips = "1.0.38" alloy-network = "1.0.38" alloy-primitives = { version = "1.4", features = ["map"] } alloy-provider = "1.0.38" +alloy-rlp = "0.3" alloy-rpc-client = "1.0.38" alloy-rpc-types-eth = "1.0.38" alloy-sol-types = "1.4" diff --git a/README.md b/README.md index f106255..2d97a5c 100644 --- a/README.md +++ b/README.md @@ -217,10 +217,10 @@ println!("installed {} bytes at {}", etched.code_size, etched.target_address); ## Benchmarks -Criterion benchmarks live in [`benches/`](benches). The offline benches are the -baseline against which the planned copy-on-write snapshot rewrite (roadmap -Pillar A) will be measured, so they exercise the real hot paths at a range of -cache sizes: +Criterion benchmarks live in [`benches/`](benches). The offline benches exercise +the current hot paths at a range of cache sizes, including the Phase 5 +copy-on-write snapshot implementation and retained deep-clone baselines where +useful for A/B comparison: | Bench | Measures | | --- | --- | diff --git a/benches/simulation.rs b/benches/simulation.rs index 82a8bd2..aca01e1 100644 --- a/benches/simulation.rs +++ b/benches/simulation.rs @@ -10,13 +10,15 @@ //! via `inject_storage_batch` (`populated_cache_layer2`), the way a fork cache //! actually holds it. For each size it benches both the COW `create_snapshot` //! and the retained `create_snapshot_deep_clone`. The deep clone is an O(total -//! state) copy, so its cost slopes up with the index size; the COW path folds -//! only the (empty) hot layer over an `Arc`-shared memoized base, so after the -//! base is warm it should stay roughly **flat** across sizes. +//! state) copy, so its cost slopes up with the index size; the COW path shares +//! the memoized base and avoids cloning total storage slots. It still scans +//! accounts and new layer-1 entries, so it should be much flatter than the deep +//! clone, especially as slots/account grows, but not strictly flat by account +//! count. //! - **`resnapshot_hot_loop`.** Warms the base with one snapshot, applies a small //! `apply_updates` layer-1 mutation, then measures `create_snapshot`. This is -//! the memoization win: ≈ O(changed) and flat across cold-index size, vs. the -//! deep clone's slope. +//! the memoization win: the COW path avoids cloning cold storage slots but +//! remains sensitive to account scans and new layer-1 entries. //! - **`overlay_fanout`.** Measures fanning one frozen snapshot out into many //! isolated simulations, comparing a fresh `EvmOverlay::new` per sim against a //! single `reset()`-recycled overlay (Pillar A.2). @@ -81,9 +83,9 @@ fn populated_cache_layer2(rt: &Runtime, accounts: usize, slots_per: usize) -> Ev /// A/B snapshot creation across cold-index sizes: the COW `create_snapshot` vs. /// the retained `create_snapshot_deep_clone`, both over a layer-2-seeded index. /// -/// The deep clone slopes up with the index; the COW path, after a warm-up -/// snapshot has memoized the base, should stay roughly flat (the hot layer is -/// empty, so it is an `Arc` handle copy plus the O(accounts) growth scan). +/// The deep clone slopes up with total slots; the COW path, after a warm-up +/// snapshot has memoized the base, avoids cloning those slots but still pays the +/// O(accounts) growth scan. fn bench_create_snapshot(c: &mut Criterion) { let rt = Runtime::new().unwrap(); let mut group = c.benchmark_group("create_snapshot"); @@ -115,7 +117,8 @@ fn bench_create_snapshot(c: &mut Criterion) { /// The memoization win: a hot re-snapshot loop. Warm the base once, apply a /// *small* layer-1 mutation, then measure `create_snapshot`. Cost should track -/// the changed state (≈ flat across cold-index size), unlike the deep clone. +/// account scanning plus new layer-1 entries, staying much flatter than the deep +/// clone as cold storage grows. fn bench_resnapshot_hot_loop(c: &mut Criterion) { let rt = Runtime::new().unwrap(); let mut group = c.benchmark_group("resnapshot_hot_loop"); diff --git a/docs/KNOWN_ISSUES.md b/docs/KNOWN_ISSUES.md index 329c2a8..9a72081 100644 --- a/docs/KNOWN_ISSUES.md +++ b/docs/KNOWN_ISSUES.md @@ -1,128 +1,116 @@ # Known issues & limitations A living triage list of bugs, smells, and limitations surfaced during the -publication-readiness review. Items here are **flagged, not fixed** — the test -suite deliberately pins *current* behavior, so changing any of these is a -conscious, reviewable decision (and a `CHANGELOG.md` entry). +publication-readiness review. Items here are either **remaining limitations** or +**recently-fixed issues kept for auditability**; behavior-changing fixes should +carry red/green tests and a `CHANGELOG.md` entry. Confidence legend: **[V]** verified against the source during review; **[R]** reported by the review and worth confirming before acting. -## Correctness / behavior to review - -1. **[V] Silent persistence failures.** `cache::save_binary_state`, - `PrefetchRegistry::save`, and `ImmutableDataCache::save` log a warning on I/O - error but return `()`, so callers cannot detect a failed write (full disk, - permissions, partial flush). Consider returning `Result<()>` (a breaking - change worth taking pre-1.0). Tested today only insofar as the happy-path - round-trip succeeds. - -2. **[V] Access-list L2 profitability uses an approximate gas model.** In - `access_list.rs`, `into_access_list_if_profitable` / `access_list_if_profitable` - estimate L1 calldata cost with hand-rolled RLP-overhead constants - (`4 * 16` per address, `16` per key, `3 * 16` for the list header). This is an - intentional heuristic, not a precise EIP-2930 serialization cost — verify it - against real serialized sizes before relying on the profitability verdict for - anything other than a rough gate. The two functions also duplicate this logic - (a maintenance hazard: a fix to one must be mirrored). - -3. **[V] Profitability swallows provider errors.** The same functions catch all - provider errors and return `Ok(None)`, which is indistinguishable from - "computed: not profitable." A caller cannot tell a skipped check (RPC down) - from a real negative. Consider a result type that distinguishes the two. - -4. **[R] `set_block` with a tag leaves `block.number` stale.** Only - `BlockId::Number(n)` syncs the `NUMBER` opcode value; pinning to a tag (e.g. - `BlockId::latest()`) leaves the previously-set number in the block env. Either - resolve tags to a concrete number at pin time or document the constraint - loudly. - -5. **[R] Duplicate custom-error selectors shadow silently.** `RevertDecoder` +## Recently fixed by `codex/phase-5-known-issues-top5` + +1. **[FIXED] Cold absolute `Account` patches no longer materialize unknown + accounts.** `StateUpdate::Account` is cold-aware: a partial patch against an + address absent from both layers is skipped, does not write a default backend + account, and is surfaced through `StateDiff.skipped_accounts`. Intentional + cold materialization is now explicit via `StateUpdate::AccountUpsert` / + `StateUpdate::account_upsert(...)`. + +2. **[FIXED] Block-context drift after re-pinning.** `set_block` now sets + `block_number` only for concrete numeric pins and clears it for tag/hash/`None` + pins. Block changes, plus non-concrete pin calls that can drift under the same + tag, clear stale `basefee`; callers refresh `NUMBER`/`BASEFEE` together with + `set_block_context` after fetching the new header. + +3. **[FIXED] Synchronous layer-2 escape hatches have an invalidating wrapper.** + Raw handles are now visibly named `unchecked_blockchain_db()` / + `unchecked_backend()`, and `EvmCache::with_blockchain_db_mut(...)` runs a + synchronous `BlockchainDb` mutation and invalidates the COW snapshot base + automatically. + +4. **[FIXED] Explicit persistence failures are observable.** + `cache::save_binary_state`, `PrefetchRegistry::save`, and + `EvmCache::flush` now return `anyhow::Result<()>`. `Drop` remains best-effort + and logs flush errors. + +5. **[FIXED] Access-list profitability uses exact EIP-2930 RLP bytes.** + Arbitrum profitability now centralizes data-gas accounting in + `access_list_rlp_data_gas(...)` and provider/pricing failures propagate as + `Err`, leaving `Ok(None)` for empty/zero-priced/unprofitable lists. + +6. **[FIXED] `simulate_call_with_balance_deltas` now returns the touched access + list.** The pre/post `balanceOf` reads and simulated call share one EVM + journal, and the method now extracts the EIP-2930 access list before + commit/revert, matching the transfer-inspector simulation path. + +7. **[FIXED] On-disk cache files carry magic bytes and a version number.** + `binary_state`, `bytecode`, `ImmutableDataCache`, and V3 tick snapshots now + write a crate-specific magic header plus version `1` before the bincode + payload. Unknown magic/version values and legacy raw-bincode files are treated + as cache misses. + +8. **[FIXED] `call_raw_with_access_list` did not revert its checkpoint on a + transact error.** Both `EvmCache::call_raw_with_access_list` and + `EvmOverlay::call_raw_with_access_list_with` now match on the `transact_one` + result and `checkpoint_revert` on **every** path (success and host error), + matching `call_raw` / `simulate_with_transfer_tracking`. A host-level transact + error no longer leaves the overlay checkpoint un-reverted. + +## Remaining open issues ranked by unexpected-result risk + +1. **[R] Duplicate custom-error selectors shadow silently.** `RevertDecoder` registration replaces an existing entry for the same 4-byte selector with no warning, so an accidental double-registration silently wins. Consider a debug-level log or a `try_register` that reports collisions. -6. **[R] ERC20 `Transfer` decoding assumes the standard layout.** `inspector.rs` +2. **[R] ERC20 `Transfer` decoding assumes the standard layout.** `inspector.rs` reads `from`/`to` from indexed topics and `value` from the first 32 data - bytes. Non-standard or packed `Transfer` encodings parse incorrectly. Also, an - address that appears as both `from` and `to` in one transfer is both - subtracted and added (a semantically-invalid self-transfer is not rejected). + bytes. Non-standard or packed `Transfer` encodings may parse incorrectly or be + skipped. A self-transfer where `from == to` nets to zero for that owner; this + is now documented at the call site. -7. **[R] Panic codes above `u64::MAX` are dropped.** `decode_solidity_panic` +3. **[V] `SystemTime::now().unwrap()` panic risk in EVM construction.** + `build_evm` / `make_local_context` (and the overlay equivalents) call + `SystemTime::now().duration_since(UNIX_EPOCH).unwrap()` when no timestamp + override is set, which panics if the system clock is before the Unix epoch. + Setting an explicit timestamp avoids it; consider a saturating fallback. + +4. **[R] Panic codes above `u64::MAX` are dropped.** `decode_solidity_panic` converts out-of-range panic codes to `None`. Real compiler-emitted panic codes - are single-byte constants, so this is benign in practice; now documented at the + are single-byte constants, so this is benign in practice and documented at the call site. -8. **[V] `simulate_call_with_balance_deltas` returns an empty access list.** It - sets `CallSimulationResult.access_list = AccessList::default()`, unlike - `simulate_with_transfer_tracking` which populates it via `extract_access_list`. - Either the field is meaningless on this path or the population was missed — - the docs now state the field is empty here; reconcile before relying on it. - -9. **[FIXED] `call_raw_with_access_list` did not revert its checkpoint on a - transact error.** Both `EvmCache::call_raw_with_access_list` and - `EvmOverlay::call_raw_with_access_list_with` now match on the `transact_one` - result and `checkpoint_revert` on **every** path (success and host error), - matching `call_raw` / `simulate_with_transfer_tracking`. A host-level transact - error no longer leaves the overlay checkpoint un-reverted. - -10. **[V] `SystemTime::now().unwrap()` panic risk in EVM construction.** - `build_evm` / `make_local_context` (and the overlay equivalents) call - `SystemTime::now().duration_since(UNIX_EPOCH).unwrap()` when no timestamp - override is set, which panics if the system clock is before the Unix epoch. - Setting an explicit timestamp avoids it; consider a saturating fallback. - -18. **[V] Cold absolute `Account` patch masks the real on-chain account.** A - *partial* absolute [`StateUpdate::Account`] patch (e.g. balance-only) applied - to an address absent from **both** cache layers writes default values for the - un-patched fields (nonce `0`, empty code) through the shared BlockchainDb - backend as authoritative — pre-empting a later RPC fetch of the real account - (`apply_account_patch` materializes the backend account on any real change, by - design / spec §5.2). This is a live-fork footgun for callers reconstructing an - account from one event field. Mitigations: fetch+seed the account first, or use - the relative `StateUpdate::BalanceDelta` / `EvmCache::modify_account_balance` - (Phase 3 §16.5), which are cold-aware (a cold target is skipped and surfaced in - `StateDiff.skipped_balances`, never materialized). A no-op patch (no field - actually changes) does **not** materialize anything (Phase 3 §16.1 fix). The - rustdoc on `apply_update` / `StateUpdate::Account` / `AccountPatch` carries a - `# Warning` to this effect. - ## Code-quality nits -11. **[V] Dead branch in `i128_to_u256`** (`cache/storage_keys.rs`): both the +5. **[V] Dead branch in `i128_to_u256`** (`cache/storage_keys.rs`): both the `value >= 0` and `else` arms evaluate the identical `U256::from(value as u128)`. The two's-complement cast is correct for both signs, so the `if`/`else` can collapse to one line (keep the explanatory comment). -12. **[R] V3 tick-snapshot keys serialize as strings.** `V3PoolTickSnapshot` +6. **[R] V3 tick-snapshot keys serialize as strings.** `V3PoolTickSnapshot` stringifies `i16`/`i32` tick/word keys for bincode, then `parse()`s them back in `to_tick_bitmap`/`to_ticks`, silently dropping any key that fails to parse. A native integer-keyed encoding would be faster and would not fail silently. -13. **[V] On-disk caches have no version header.** `binary_state`, `bytecode`, - `metadata` (`ImmutableDataCache`), and `tick_snapshot` all persist raw bincode - with no magic bytes or version field, so a struct-layout change silently - invalidates every existing cache file (decoded as a miss). A version header - would enable detection/migration. - -14. **[R] Balancer pool id keyed by `Debug` formatting.** `ImmutableDataCache` +7. **[R] Balancer pool id keyed by `Debug` formatting.** `ImmutableDataCache` keys `balancer_pools` by `format!("{:?}", pool_id)`. `Debug` output is not a stable encoding contract; a hex encoding would be safer for a persisted key. ## API ergonomics -15. **[R] `snapshot()` vs `create_snapshot()`.** `snapshot()` returns a low-level +8. **[R] `snapshot()` vs `create_snapshot()`.** `snapshot()` returns a low-level `revm::database::Cache` for in-place `restore()`; `create_snapshot()` returns an `Arc` for cross-thread fan-out. The names don't convey the difference. Docs now cross-reference them (see the rustdoc), but a rename could be considered pre-1.0. -16. **[R] Process-global cache speed mode.** `set_cache_speed_mode` / +9. **[R] Process-global cache speed mode.** `set_cache_speed_mode` / `cache_speed_mode` are a process-wide `static`, so two caches in one process cannot tune concurrency independently. Phase 1 moved configuration toward per-instance (`EvmCacheBuilder::cache_config`); the global setter remains. -17. **[V] `SpeculativeSim` consumption contract.** Both `validate()` and +10. **[V] `SpeculativeSim` consumption contract.** Both `validate()` and `into_optimistic()` take `self` by value, so double-consumption is unreachable under normal ownership. Internally `validate()` uses `.expect("validation handle taken twice")` (defensive) while `into_optimistic()` no-ops if the @@ -148,29 +136,35 @@ Confidence legend: **[V]** verified against the source during review; kept as the A/B benchmark baseline and the read-equivalence reference; the `create_snapshot` group in `benches/simulation.rs` measures both. Decisions and the cost model are in [`phase-5-spec.md`](phase-5-spec.md) / `ROADMAP.md`. -- **[V] Memoized-base staleness at the layer-2 escape hatches (Phase 5).** The +- **Layer-2 unchecked accessors remain an explicit contract boundary (Phase 5).** The snapshot base's growth scan is count/absence-based, which is sufficient for the supported writers: the crate's own mutators (`apply_update`, `inject_storage_batch`, the `inject_*` helpers, purges, code overrides) explicitly mark the base dirty, and the `foundry-fork-db` `SharedBackend` lazy fetch is append-only at a fixed block (it only inserts on a cache miss, never overwrites in place — a load-bearing - invariant noted in `refresh_base`). The one gap, surfaced by the Phase 5 - adversarial review: a **direct, out-of-band write through the public - `blockchain_db()` / `backend()` handles** that *overwrites an existing slot value - at an unchanged slot count* is invisible to the scan, so a subsequent - `create_snapshot` may reuse a stale base (`create_snapshot_deep_clone` always - re-reads and would diverge). This is a contract boundary, not an internal bug — no - in-crate path triggers it, and both accessors are documented as bypassing the - two-layer model. Mitigation: call the new - [`EvmCache::invalidate_snapshot_base`] after any direct layer-2 write through those - handles (or re-pin via `set_block`); the rustdoc on both accessors and the hook - carries this warning, and `tests/cow_snapshot.rs` - (`invalidate_snapshot_base_rehonest_after_escape_hatch_write`) pins it. + invariant noted in `refresh_base`). Direct out-of-band writes through the + `unchecked_blockchain_db()` / `unchecked_backend()` handles still bypass the + normal write funnel by design. For synchronous `BlockchainDb` map writes, prefer + [`EvmCache::with_blockchain_db_mut`], which invalidates the base automatically + after the closure returns. If using the unchecked handle directly, call + [`EvmCache::invalidate_snapshot_base`] after the write lands and before the next + snapshot (or re-pin via `set_block`). For + `SharedBackend::insert_or_update_storage` / `insert_or_update_address`, the call + only enqueues work on the backend handler; `invalidate_snapshot_base()` does not + wait for that queued update. First synchronize or read back until the expected + value is visible in `BlockchainDb` / through the backend, then invalidate before + creating the snapshot. The rustdoc on both accessors and the hook carries this + warning, and `tests/cow_snapshot.rs` + (`invalidate_snapshot_base_rehonest_after_escape_hatch_write`, + `invalidate_snapshot_base_rehonest_after_existing_account_write`, + `with_blockchain_db_mut_rehonest_after_storage_overwrite`, + `with_blockchain_db_mut_rehonest_after_account_overwrite`) + pins it. - **`protocols` not yet extracted.** The DeFi surface is feature-gated but still - in-crate; `cargo test --no-default-features` is not yet supported because some - unit tests assume the default feature. Extraction into `evm-amm-state` is - planned (roadmap), blocked partly by `ImmutableDataCache` coupling generic - token-decimals with V2/V3/Balancer pool metadata. + in-crate. The generic core builds and tests with `--no-default-features`, but + extraction into `evm-amm-state` is still planned (roadmap), blocked partly by + `ImmutableDataCache` coupling generic token-decimals with V2/V3/Balancer pool + metadata. - **Event-driven sync (roadmap Pillar B) — reader/writer halves done; live WS transport is not.** The Phase 3 **writer half** (`StateUpdate` + `apply_update`/`apply_updates`) and the Phase 4 **reader half** (the `events` diff --git a/docs/phase-2-spec.md b/docs/phase-2-spec.md index c61f19c..baff6b2 100644 --- a/docs/phase-2-spec.md +++ b/docs/phase-2-spec.md @@ -54,7 +54,7 @@ evaluation sims only. `storage_batch_fetcher() -> Option<&StorageBatchFetchFn>`, `inject_storage_batch(&[(Address,U256,U256)])`, `purge_pool_storage`, `purge_pool_slots`, `call_raw_with`/`TxConfig`, `CallSimulationResult`, - `blockchain_db()`, `db_mut()`. + `unchecked_blockchain_db()`, `db_mut()`. - `cache::EvmOverlay` / `cache::EvmSnapshot` (`overlay.rs`/`snapshot.rs`): `EvmOverlay::new(Arc, Option)`, `call_raw`, `simulate_with_transfer_tracking`. `EvmOverlay` is `Send`. diff --git a/docs/phase-3-spec.md b/docs/phase-3-spec.md index 199eaf5..e51886c 100644 --- a/docs/phase-3-spec.md +++ b/docs/phase-3-spec.md @@ -358,7 +358,9 @@ in a new `tests/state_update.rs` (reuse `tests/common`). `nonce/code_hash == None`. 6. **Account code patch:** patch code; assert `code_hash` recomputed (`Bytecode::hash_slow`), `code_hash` delta recorded; balance/nonce preserved. -7. **Account create:** patch an absent account → materialized with patched fields. +7. **Cold account patch:** patch an absent account → skipped and surfaced in + `StateDiff.skipped_accounts`; explicit `AccountUpsert` materializes with + patched fields. 8. **Purge Account / AllStorage / Slots:** correct layers cleared; `PurgeRecord` counts (`slots_removed`, `account_removed`) correct on both layers. 9. **`apply_updates` fold + merge:** a mixed batch (Slot, Account, Purge) → @@ -645,7 +647,7 @@ the `SlotDelta`/`modify_slot` base read (HIGH) and `apply_slot`'s `old`/predicat **and** no backend value) stays skip-and-surface. Add a test for the hot-zero case (it is currently the untested seam between Decision-4 skip and apply). -### 16.1 No-op `Account` patch must not materialize a backend account (audit LOW) +### 16.1 No-op / cold `Account` patch must not materialize a backend account (audit LOW; tightened in Phase 5) `apply_account_patch` (src/cache/mod.rs ~1331-1340) writes the patched `AccountInfo` into the backend **unconditionally**, so an all-`None` (or @@ -654,23 +656,21 @@ otherwise no-change) patch on an address absent from both layers inserts diff — breaking no-op parity with the Slot path and (per the cold-account hazard) masking a future RPC fetch. **Fix (LOCKED):** compute the change first; **only write-through when at least one field actually changes** (i.e. skip both layer -writes and return `None` when the patched `info` equals the loaded base). A real -field change on an absent address still materializes the backend account (the -existing intended behavior — keep `apply_account_patch_materializes_absent_account` -green). Add a no-op idempotence test (patching balance to its current value ⇒ -empty diff, no backend account materialized). - -### 16.2 Cold absolute-`Account`-patch hazard — document (audit LOW) - -A *partial* absolute `Account` patch on a cold (un-fetched) address writes default -nonce/code through the shared backend, masking the real on-chain account. This is -spec-locked §5.2 behavior, **not** changed here, but it is an undocumented -live-fork footgun. **Fix (LOCKED, docs only):** add a `### Known issues` entry in -`docs/KNOWN_ISSUES.md` and a prominent `# Warning` doc paragraph on -`apply_update` / `StateUpdate::Account` / `AccountPatch` stating that a partial -patch on an address absent from both cache layers writes default nonce/code as -authoritative (pre-empting RPC), so callers must fetch+seed the account first, or -use `StateUpdate::BalanceDelta` (§16.5) for relative native-balance tracking. +writes and return `None` when the patched `info` equals the loaded base). +Phase 5 tightened the cold-account contract further: a real field change on an +address absent from both layers now skips and records `SkippedAccountPatch` in +`StateDiff.skipped_accounts`; explicit materialization uses +`StateUpdate::AccountUpsert`. Add no-op and cold-skip idempotence tests (patching +balance to its current value ⇒ empty diff; cold balance patch ⇒ no backend account +materialized). + +### 16.2 Cold absolute-`Account`-patch hazard — fixed in Phase 5 (audit LOW) + +A *partial* absolute `Account` patch on a cold (un-fetched) address used to write +default nonce/code through the shared backend, masking the real on-chain account. +Phase 5 changed this contract: `StateUpdate::Account` is cold-aware and records a +`SkippedAccountPatch` instead; callers that intentionally want a synthetic/default +account use `StateUpdate::AccountUpsert`. ### 16.3 `serde` on the vocabulary (audit HIGH gap) @@ -778,7 +778,7 @@ Add tests (in `tests/state_update.rs` unless noted). Each must assert the materialized where the spec says none should be (mirror `apply_slot_no_overlay_account_is_not_materialized`). - **Backend-only account patch:** seed an account only in the backend - (`blockchain_db().accounts().write().insert`), patch balance, assert + (`unchecked_blockchain_db().accounts().write().insert`), patch balance, assert `AccountChange.balance == Some((old,new))`, backend updated, overlay still absent. - **Nonce-only** and **multi-field (balance+nonce+code)** patches: assert the respective `AccountChange` fields are `Some`/`None` correctly. diff --git a/examples/prefetch_registry.rs b/examples/prefetch_registry.rs index 3c57366..690092b 100644 --- a/examples/prefetch_registry.rs +++ b/examples/prefetch_registry.rs @@ -19,7 +19,7 @@ use alloy_primitives::{Address, U256}; use evm_fork_cache::StorageAccessList; use evm_fork_cache::prefetch_registry::PrefetchRegistry; -fn main() { +fn main() -> anyhow::Result<()> { let pool = Address::repeat_byte(0xAA); let vault_a = Address::repeat_byte(0x01); let vault_b = Address::repeat_byte(0x02); @@ -45,7 +45,7 @@ fn main() { // Persist to disk (bincode) and reload — the shape survives the round trip. let path = std::env::temp_dir().join("evm_fork_cache_example_prefetch.bin"); - registry.save(&path); + registry.save(&path)?; let loaded = PrefetchRegistry::load(&path); let aggregated = loaded.phase_slots("pool_refresh"); @@ -61,4 +61,5 @@ fn main() { ); let _ = std::fs::remove_file(&path); + Ok(()) } diff --git a/src/access_list.rs b/src/access_list.rs index a84c99e..bf9df24 100644 --- a/src/access_list.rs +++ b/src/access_list.rs @@ -15,8 +15,9 @@ use alloy_eips::eip2930::{AccessList, AccessListItem}; use alloy_network::Network; use alloy_primitives::{Address, B256, U256, address}; use alloy_provider::Provider; +use alloy_rlp::Encodable; use alloy_sol_types::{SolCall, sol}; -use anyhow::Result; +use anyhow::{Context as _, Result}; use revm::context::result::ExecutionResult; use tracing::{debug, info}; @@ -141,36 +142,18 @@ impl SmartAccessList { /// - **L2 savings**: `100 gas * entry_count * perArbGas`, where each address /// and each storage key counts as one entry (the EIP-2929 warm-vs-cold /// access discount). - /// - **L1 cost**: `l1_data_gas * l1_base_fee`, where `l1_data_gas` sums the - /// per-byte calldata gas ([`l1_data_gas_for_bytes`]) of every address and - /// key plus a fixed RLP-framing surcharge. - /// - /// # Cost model is approximate - /// - /// The RLP-overhead constants — roughly `4 * 16` gas per address entry, - /// `16` gas per storage key, and `3 * 16` gas for the top-level list headers - /// — are a deliberate **approximation**, not the exact EIP-2930 RLP - /// serialization cost. They assume worst-case non-zero framing bytes and do - /// not account for the real RLP length-prefix sizing, address/key sharing, - /// or rollup-specific compression. Treat this as a rough profitability gate, - /// not a precise gas accounting: a list near the break-even point may be - /// classified either way. + /// - **L1 cost**: `l1_data_gas * l1_base_fee`, where `l1_data_gas` is the + /// exact per-byte calldata gas ([`l1_data_gas_for_bytes`]) of the EIP-2930 + /// RLP-encoded access list. /// /// # Errors /// - /// Returns `Err` only if the call wrapper itself surfaces a non-recoverable - /// error; in practice provider/pricing failures do **not** error. + /// Returns `Err` if the provider/pricing queries fail. /// /// Returns `Ok(None)` when: /// - the list is empty, - /// - the `ArbGasInfo` pricing or L1-base-fee query fails (the error is logged - /// at `debug` and swallowed — see below), /// - either the L2 or L1 gas price reads as zero, or /// - the estimated L1 cost meets or exceeds the L2 savings (not profitable). - /// - /// A `None` returned because a provider query failed is **indistinguishable** - /// from a `None` returned because the list was genuinely unprofitable: both - /// surface as a skipped access list, not as an error. pub async fn into_access_list_if_profitable( self, provider: &P, @@ -182,21 +165,15 @@ impl SmartAccessList { // Query ArbGasInfo for current pricing let arb = ArbGasInfo::new(ARB_GAS_INFO, provider); let prices_call = arb.getPricesInWei(); - let prices = match prices_call.call().await { - Ok(p) => p, - Err(e) => { - debug!(error = %e, "Failed to query ArbGasInfo prices, skipping access list"); - return Ok(None); - } - }; + let prices = prices_call + .call() + .await + .context("failed to query ArbGasInfo prices for access-list profitability")?; let l1_fee_call = arb.getL1BaseFeeEstimate(); - let l1_base_fee = match l1_fee_call.call().await { - Ok(fee) => fee, - Err(e) => { - debug!(error = %e, "Failed to query L1 base fee, skipping access list"); - return Ok(None); - } - }; + let l1_base_fee = l1_fee_call + .call() + .await + .context("failed to query ArbGasInfo L1 base fee for access-list profitability")?; let l2_gas_price = prices.perArbGas; @@ -205,46 +182,14 @@ impl SmartAccessList { return Ok(None); } - // Calculate aggregate L2 savings and L1 cost - let mut total_entries: u64 = 0; - let mut total_l1_data_gas: u64 = 0; - - for item in &self.items { - total_entries += 1; - total_l1_data_gas += l1_data_gas_for_bytes(item.address.as_slice()); - // RLP overhead per address entry (~3-4 bytes, assume non-zero) - total_l1_data_gas += 4 * 16; - - for key in &item.storage_keys { - total_entries += 1; - total_l1_data_gas += l1_data_gas_for_bytes(key.as_slice()); - // RLP length prefix (1 byte, non-zero) - total_l1_data_gas += 16; - } - } - // Top-level RLP list headers (~3 bytes) - total_l1_data_gas += 3 * 16; - - // L2 savings: 100 gas per entry × L2 gas price - let l2_savings_wei = U256::from(total_entries) * U256::from(100) * l2_gas_price; - // L1 cost: serialized data gas × L1 base fee - let l1_cost_wei = U256::from(total_l1_data_gas) * l1_base_fee; - - let profitable = l2_savings_wei > l1_cost_wei; - - info!( - entries = total_entries, - items = self.items.len(), - l2_savings_wei = %l2_savings_wei, - l1_cost_wei = %l1_cost_wei, - l2_gas_price_gwei = %format_gwei(l2_gas_price), - l1_base_fee_gwei = %format_gwei(l1_base_fee), - profitable, - "Access list profitability check" - ); - - if profitable { - Ok(Some(AccessList(self.items))) + let access_list = AccessList(self.items); + if log_access_list_profitability( + &access_list, + l2_gas_price, + l1_base_fee, + "Access list profitability check", + ) { + Ok(Some(access_list)) } else { Ok(None) } @@ -261,31 +206,14 @@ impl SmartAccessList { /// [`SmartAccessList::into_access_list_if_profitable`] for a pre-built /// [`AccessList`]; the two share the same cost model and break-even comparison. /// -/// # Cost model is approximate -/// -/// As with [`SmartAccessList::into_access_list_if_profitable`], the L1 cost is -/// estimated from per-byte calldata gas ([`l1_data_gas_for_bytes`]) plus fixed -/// RLP-framing surcharges (`4 * 16` gas per address, `16` gas per key, `3 * 16` -/// gas for the top-level headers). Those framing constants are an -/// **approximation**, not the exact EIP-2930 RLP serialization cost: they assume -/// worst-case non-zero bytes and ignore real length-prefix sizing and -/// rollup-specific compression. Treat the result as a rough profitability gate. -/// /// # Errors /// -/// Returns `Err` only if the call wrapper itself surfaces a non-recoverable -/// error; in practice provider/pricing failures do **not** error. +/// Returns `Err` if the provider/pricing queries fail. /// /// Returns `Ok(None)` when: /// - the list is empty, -/// - the `ArbGasInfo` pricing or L1-base-fee query fails (the error is logged at -/// `debug` and swallowed), /// - either the L2 or L1 gas price reads as zero, or /// - the estimated L1 cost meets or exceeds the L2 savings (not profitable). -/// -/// A `None` returned because a provider query failed is **indistinguishable** -/// from a `None` returned because the list was genuinely unprofitable: both -/// surface as a skipped access list, not as an error. pub async fn access_list_if_profitable( access_list: AccessList, provider: &P, @@ -296,20 +224,16 @@ pub async fn access_list_if_profitable( // Query ArbGasInfo for current pricing let arb = ArbGasInfo::new(ARB_GAS_INFO, provider); - let prices = match arb.getPricesInWei().call().await { - Ok(p) => p, - Err(e) => { - debug!(error = %e, "Failed to query ArbGasInfo prices, skipping access list"); - return Ok(None); - } - }; - let l1_base_fee = match arb.getL1BaseFeeEstimate().call().await { - Ok(fee) => fee, - Err(e) => { - debug!(error = %e, "Failed to query L1 base fee, skipping access list"); - return Ok(None); - } - }; + let prices = arb + .getPricesInWei() + .call() + .await + .context("failed to query ArbGasInfo prices for access-list profitability")?; + let l1_base_fee = arb + .getL1BaseFeeEstimate() + .call() + .await + .context("failed to query ArbGasInfo L1 base fee for access-list profitability")?; let l2_gas_price = prices.perArbGas; @@ -318,45 +242,12 @@ pub async fn access_list_if_profitable( return Ok(None); } - // Calculate aggregate L2 savings and L1 cost - let mut total_entries: u64 = 0; - let mut total_l1_data_gas: u64 = 0; - - for item in &access_list.0 { - total_entries += 1; - total_l1_data_gas += l1_data_gas_for_bytes(item.address.as_slice()); - // RLP overhead per address entry (~3-4 bytes, assume non-zero) - total_l1_data_gas += 4 * 16; - - for key in &item.storage_keys { - total_entries += 1; - total_l1_data_gas += l1_data_gas_for_bytes(key.as_slice()); - // RLP length prefix (1 byte, non-zero) - total_l1_data_gas += 16; - } - } - // Top-level RLP list headers (~3 bytes) - total_l1_data_gas += 3 * 16; - - // L2 savings: 100 gas per entry × L2 gas price - let l2_savings_wei = U256::from(total_entries) * U256::from(100) * l2_gas_price; - // L1 cost: serialized data gas × L1 base fee - let l1_cost_wei = U256::from(total_l1_data_gas) * l1_base_fee; - - let profitable = l2_savings_wei > l1_cost_wei; - - info!( - entries = total_entries, - items = access_list.0.len(), - l2_savings_wei = %l2_savings_wei, - l1_cost_wei = %l1_cost_wei, - l2_gas_price_gwei = %format_gwei(l2_gas_price), - l1_base_fee_gwei = %format_gwei(l1_base_fee), - profitable, - "Simulation access list profitability check" - ); - - if profitable { + if log_access_list_profitability( + &access_list, + l2_gas_price, + l1_base_fee, + "Simulation access list profitability check", + ) { Ok(Some(access_list)) } else { Ok(None) @@ -483,6 +374,49 @@ pub fn l1_data_gas_for_bytes(data: &[u8]) -> u64 { .sum() } +/// Exact L1 calldata gas for the EIP-2930 RLP encoding of an access list. +pub fn access_list_rlp_data_gas(access_list: &AccessList) -> u64 { + let mut encoded = Vec::with_capacity(access_list.length()); + access_list.encode(&mut encoded); + l1_data_gas_for_bytes(&encoded) +} + +fn access_list_entry_count(access_list: &AccessList) -> u64 { + access_list + .0 + .iter() + .map(|item| 1 + item.storage_keys.len() as u64) + .sum() +} + +fn log_access_list_profitability( + access_list: &AccessList, + l2_gas_price: U256, + l1_base_fee: U256, + message: &'static str, +) -> bool { + let total_entries = access_list_entry_count(access_list); + let total_l1_data_gas = access_list_rlp_data_gas(access_list); + let l2_savings_wei = U256::from(total_entries) * U256::from(100) * l2_gas_price; + let l1_cost_wei = U256::from(total_l1_data_gas) * l1_base_fee; + let profitable = l2_savings_wei > l1_cost_wei; + + info!( + entries = total_entries, + items = access_list.0.len(), + l1_data_gas = total_l1_data_gas, + l2_savings_wei = %l2_savings_wei, + l1_cost_wei = %l1_cost_wei, + l2_gas_price_gwei = %format_gwei(l2_gas_price), + l1_base_fee_gwei = %format_gwei(l1_base_fee), + profitable, + check = message, + "Access list profitability check" + ); + + profitable +} + /// Filter already-warm and excluded addresses from an access list, then apply /// it to the transaction request. /// @@ -571,4 +505,53 @@ mod tests { let addr = Address::repeat_byte(0xFF); assert_eq!(l1_data_gas_for_bytes(addr.as_slice()), 320); } + + #[test] + fn access_list_rlp_data_gas_uses_exact_eip2930_encoding() { + let access_list = AccessList(vec![AccessListItem { + address: Address::ZERO, + storage_keys: Vec::new(), + }]); + + // RLP([[zero_address, []]]) = d7 d6 94 <20 zero bytes> c0. + // Four non-zero framing bytes cost 64 gas; twenty zero address bytes cost + // 80 gas. The old fixed-overhead approximation returned 192. + assert_eq!(access_list_rlp_data_gas(&access_list), 144); + } + + #[tokio::test] + async fn access_list_profitability_provider_error_returns_err() { + use alloy_network::Ethereum; + use alloy_provider::RootProvider; + use alloy_rpc_client::RpcClient; + use alloy_transport::mock::Asserter; + + let provider = RootProvider::::new(RpcClient::mocked(Asserter::new())); + let access_list = AccessList(vec![AccessListItem { + address: Address::repeat_byte(0xAA), + storage_keys: Vec::new(), + }]); + + let err = access_list_if_profitable(access_list, &provider) + .await + .expect_err("provider failures must be distinguishable from unprofitable lists"); + assert!( + err.to_string().contains("ArbGasInfo") || err.to_string().contains("provider"), + "unexpected error: {err:#}" + ); + } + + #[tokio::test] + async fn access_list_profitability_empty_list_still_returns_none() { + use alloy_network::Ethereum; + use alloy_provider::RootProvider; + use alloy_rpc_client::RpcClient; + use alloy_transport::mock::Asserter; + + let provider = RootProvider::::new(RpcClient::mocked(Asserter::new())); + let result = access_list_if_profitable(AccessList::default(), &provider) + .await + .expect("empty list must not query provider"); + assert!(result.is_none()); + } } diff --git a/src/cache/binary_state.rs b/src/cache/binary_state.rs index 1deef61..ccfb3d0 100644 --- a/src/cache/binary_state.rs +++ b/src/cache/binary_state.rs @@ -5,20 +5,25 @@ //! and write a compact binary file. On load, we populate BlockchainDb directly, //! then seed bytecodes from the separate bytecodes.bin cache. //! -//! The file format is raw bincode with no version header or magic bytes, so it -//! is not migratable: a cache written by a build with a different struct layout -//! decodes as a failure (cache miss) rather than being upgraded in place. +//! The file format is a tiny crate-specific envelope (magic bytes + version) +//! followed by bincode payload. Unknown magic/version values are cache misses. use std::path::Path; use std::time::Instant; use alloy_primitives::map::HashMap; use alloy_primitives::{Address, B256, U256}; +use anyhow::{Context as _, Result}; use foundry_fork_db::BlockchainDb; use revm::state::AccountInfo; use serde::{Deserialize, Serialize}; use tracing::{debug, warn}; +use super::versioned; + +const BINARY_STATE_MAGIC: &[u8; 8] = b"EFCSTAT\0"; +const BINARY_STATE_VERSION: u32 = 1; + /// Binary-serializable EVM state. Stores accounts without bytecode (bytecodes /// are loaded separately from bytecodes.bin) and all storage slots. #[derive(Serialize, Deserialize)] @@ -42,16 +47,14 @@ struct BinaryAccountInfo { /// and persisted separately to `bytecodes.bin`; the saved account info keeps /// only the `code_hash`. /// -/// Errors are logged at `warn` level and otherwise swallowed: serialization -/// failures, parent-directory creation failures, and write failures all return -/// without signalling to the caller, so a failed save is indistinguishable from -/// a successful one at the call site. +/// Returns an error if serialization, parent-directory creation, or writing +/// fails, so explicit flush callers can distinguish a successful save from a +/// stale or missing on-disk cache. /// -/// The on-disk format is raw bincode with no version header, so it is not -/// forward/backward compatible: a file written by a build with a different -/// layout will fail to decode on load (treated as a cache miss) rather than +/// The on-disk format carries magic bytes and a version number before the +/// bincode payload. Unknown versions are treated as a cache miss rather than /// being migrated. -pub fn save_binary_state(blockchain_db: &BlockchainDb, path: &Path) { +pub fn save_binary_state(blockchain_db: &BlockchainDb, path: &Path) -> Result<()> { let start = Instant::now(); let accounts: Vec<(Address, BinaryAccountInfo)> = blockchain_db @@ -79,27 +82,28 @@ pub fn save_binary_state(blockchain_db: &BlockchainDb, path: &Path) { let state = BinaryEvmState { accounts, storage }; - match bincode::serialize(&state) { - Ok(data) => { - if let Some(parent) = path.parent() { - let _ = std::fs::create_dir_all(parent); - } - match std::fs::write(path, &data) { - Ok(()) => { - let ms = start.elapsed().as_millis(); - debug!( - accounts = state.accounts.len(), - storage_contracts = state.storage.len(), - bytes = data.len(), - save_ms = ms, - "Saved binary EVM state" - ); - } - Err(e) => warn!(error = %e, "Failed to write binary EVM state"), - } - } - Err(e) => warn!(error = %e, "Failed to serialize binary EVM state"), + let data = versioned::encode( + BINARY_STATE_MAGIC, + BINARY_STATE_VERSION, + &state, + "binary EVM state", + )?; + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("failed to create binary EVM state directory {parent:?}"))?; } + std::fs::write(path, &data) + .with_context(|| format!("failed to write binary EVM state to {path:?}"))?; + + let ms = start.elapsed().as_millis(); + debug!( + accounts = state.accounts.len(), + storage_contracts = state.storage.len(), + bytes = data.len(), + save_ms = ms, + "Saved binary EVM state" + ); + Ok(()) } /// Load binary EVM state and populate the BlockchainDb. @@ -109,9 +113,8 @@ pub fn save_binary_state(blockchain_db: &BlockchainDb, path: &Path) { /// Bytecodes should be seeded separately from bytecodes.bin. /// /// Returns `false` (rather than erroring) when `path` cannot be read or its -/// contents fail to decode as the expected bincode layout; a decode failure is -/// logged at `warn` level. Because the format carries no version header, a file -/// written by an incompatible build is reported as a decode failure here. +/// contents fail the magic/version check or fail to decode as the expected +/// bincode layout; failures are logged at `warn` level. pub fn load_binary_state(blockchain_db: &BlockchainDb, path: &Path) -> bool { let start = Instant::now(); @@ -120,12 +123,14 @@ pub fn load_binary_state(blockchain_db: &BlockchainDb, path: &Path) -> bool { Err(_) => return false, }; - let state: BinaryEvmState = match bincode::deserialize(&data) { - Ok(s) => s, - Err(e) => { - warn!(?e, "Failed to decode binary EVM state, starting fresh"); - return false; - } + let Some(state) = versioned::decode::( + &data, + BINARY_STATE_MAGIC, + BINARY_STATE_VERSION, + "binary EVM state", + ) else { + warn!("Failed to decode binary EVM state, starting fresh"); + return false; }; let account_count = state.accounts.len(); @@ -229,8 +234,18 @@ mod tests { } // Save - save_binary_state(&db, &path); + save_binary_state(&db, &path).expect("save binary state"); assert!(path.exists()); + let bytes = std::fs::read(&path).expect("read saved state"); + assert!( + bytes.starts_with(b"EFCSTAT\0"), + "binary state cache must carry a magic header" + ); + assert_eq!( + &bytes[8..12], + &1u32.to_le_bytes(), + "binary state cache must carry an explicit version" + ); // Load into a fresh db let meta2 = BlockchainDbMeta::default(); @@ -265,6 +280,24 @@ mod tests { let _ = std::fs::remove_dir(&dir); } + #[test] + fn save_binary_state_reports_write_failures() { + let dir = std::env::temp_dir().join("evm_fork_cache_test_binary_state_write_error"); + let _ = std::fs::remove_dir_all(&dir); + let _ = std::fs::remove_file(&dir); + std::fs::write(&dir, b"not a directory").expect("create file path conflict"); + + let db = BlockchainDb::new(BlockchainDbMeta::default(), None); + let path = dir.join("state.bin"); + let err = save_binary_state(&db, &path).expect_err("save must report write failure"); + assert!( + err.to_string().contains("directory") || err.to_string().contains("Not a directory"), + "unexpected error: {err:#}" + ); + + let _ = std::fs::remove_file(&dir); + } + #[test] fn test_load_missing_file_returns_false() { let meta = BlockchainDbMeta::default(); @@ -289,4 +322,25 @@ mod tests { let _ = std::fs::remove_file(&path); let _ = std::fs::remove_dir(&dir); } + + #[test] + fn load_legacy_raw_bincode_returns_false() { + let dir = std::env::temp_dir().join("evm_fork_cache_test_binary_state_legacy"); + let path = dir.join("legacy.bin"); + let _ = std::fs::create_dir_all(&dir); + let legacy = BinaryEvmState { + accounts: Vec::new(), + storage: Vec::new(), + }; + std::fs::write(&path, bincode::serialize(&legacy).unwrap()).unwrap(); + + let db = BlockchainDb::new(BlockchainDbMeta::default(), None); + assert!( + !load_binary_state(&db, &path), + "unversioned legacy bincode must be treated as a cache miss" + ); + + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_dir(&dir); + } } diff --git a/src/cache/bytecode.rs b/src/cache/bytecode.rs index a894a58..1ef00bc 100644 --- a/src/cache/bytecode.rs +++ b/src/cache/bytecode.rs @@ -6,10 +6,9 @@ //! entries are used to re-seed the `code` of accounts that were restored //! without it. //! -//! Each entry's bytes are hex-encoded for the serde representation, but the -//! file is written as raw bincode with no version header, so a cache written by -//! an incompatible build fails to decode (cache miss) rather than being -//! migrated. +//! Each entry's bytes are hex-encoded for the serde representation. The file is +//! written as a crate-specific versioned envelope followed by bincode payload, so +//! incompatible versions are detected as cache misses. use std::collections::HashMap; use std::path::Path; @@ -18,7 +17,11 @@ use alloy_primitives::Address; use anyhow::Result; use foundry_fork_db::BlockchainDb; use serde::{Deserialize, Serialize}; -use tracing::warn; + +use super::versioned; + +const BYTECODE_CACHE_MAGIC: &[u8; 8] = b"EFCBYTE\0"; +const BYTECODE_CACHE_VERSION: u32 = 1; /// Serializable bytecode cache entry. #[derive(Debug, Clone, Serialize, Deserialize)] @@ -38,15 +41,16 @@ pub(crate) struct BytecodeCache { impl BytecodeCache { /// Load bytecode cache from disk (binary format). /// - /// Returns `None` if `path` cannot be read or its contents fail to decode as - /// bincode for this type; a decode failure is logged at `warn` level. The - /// format carries no version header, so a file from an incompatible build is - /// reported as `None`. + /// Returns `None` if `path` cannot be read, fails the magic/version check, or + /// fails to decode as bincode for this type. pub(crate) fn load(path: &Path) -> Option { let data = std::fs::read(path).ok()?; - bincode::deserialize(&data) - .inspect_err(|e| warn!("Failed to parse bytecode cache (bincode): {}", e)) - .ok() + versioned::decode( + &data, + BYTECODE_CACHE_MAGIC, + BYTECODE_CACHE_VERSION, + "bytecode cache", + ) } /// Save bytecode cache to disk (binary format). @@ -62,7 +66,12 @@ impl BytecodeCache { if let Some(parent) = path.parent() { std::fs::create_dir_all(parent)?; } - let data = bincode::serialize(self)?; + let data = versioned::encode( + BYTECODE_CACHE_MAGIC, + BYTECODE_CACHE_VERSION, + self, + "bytecode cache", + )?; std::fs::write(path, data)?; Ok(()) } @@ -117,6 +126,16 @@ mod tests { }, ); cache.save(&path).expect("save bytecode cache"); + let bytes = std::fs::read(&path).expect("read saved bytecode cache"); + assert!( + bytes.starts_with(b"EFCBYTE\0"), + "bytecode cache must carry a magic header" + ); + assert_eq!( + &bytes[8..12], + &1u32.to_le_bytes(), + "bytecode cache must carry an explicit version" + ); let loaded = BytecodeCache::load(&path).expect("load bytecode cache"); assert_eq!( @@ -128,6 +147,26 @@ mod tests { let _ = std::fs::remove_dir_all(path.parent().unwrap()); } + #[test] + fn load_legacy_raw_bincode_is_none() { + let path = temp_path("legacy"); + let mut cache = BytecodeCache::default(); + cache.contracts.insert( + Address::repeat_byte(0x42), + BytecodeCacheEntry { + bytecode: vec![0x60, 0x00], + }, + ); + std::fs::write(&path, bincode::serialize(&cache).unwrap()).expect("write legacy cache"); + + assert!( + BytecodeCache::load(&path).is_none(), + "unversioned legacy bincode must be treated as a cache miss" + ); + + let _ = std::fs::remove_dir_all(path.parent().unwrap()); + } + #[test] fn load_missing_file_is_none() { assert!(BytecodeCache::load(std::path::Path::new("/nonexistent/bytecodes.bin")).is_none()); diff --git a/src/cache/metadata.rs b/src/cache/metadata.rs index 6ab9472..670de08 100644 --- a/src/cache/metadata.rs +++ b/src/cache/metadata.rs @@ -12,10 +12,14 @@ use std::path::{Path, PathBuf}; use alloy_primitives::{Address, B256, U256}; use anyhow::Result; use serde::{Deserialize, Serialize}; -use tracing::warn; use std::collections::HashSet; +use super::versioned; + +const IMMUTABLE_CACHE_MAGIC: &[u8; 8] = b"EFCMETA\0"; +const IMMUTABLE_CACHE_VERSION: u32 = 1; + /// Configuration for disk-based caching of EVM state. /// /// Enables on-disk persistence of fetched fork state. Cache files are laid out @@ -158,18 +162,17 @@ pub struct ImmutableDataCache { impl ImmutableDataCache { /// Load immutable data cache from disk (binary format). /// - /// Returns `None` if `path` cannot be read or the contents are not valid - /// bincode for this type (a parse failure is logged at `warn` level and - /// swallowed). Callers should treat `None` as "no cache yet" and start fresh. - /// - /// Note: the on-disk format is bincode with no version header, so a cache - /// written by an incompatible build deserializes as a parse failure (`None`) - /// rather than being migrated. + /// Returns `None` if `path` cannot be read, fails the magic/version check, or + /// the payload is not valid bincode for this type. Callers should treat + /// `None` as "no cache yet" and start fresh. pub fn load(path: &Path) -> Option { let data = std::fs::read(path).ok()?; - bincode::deserialize(&data) - .inspect_err(|e| warn!("Failed to parse immutable data cache (bincode): {}", e)) - .ok() + versioned::decode( + &data, + IMMUTABLE_CACHE_MAGIC, + IMMUTABLE_CACHE_VERSION, + "immutable data cache", + ) } /// Save immutable data cache to disk (binary format). @@ -185,7 +188,12 @@ impl ImmutableDataCache { if let Some(parent) = path.parent() { std::fs::create_dir_all(parent)?; } - let data = bincode::serialize(self)?; + let data = versioned::encode( + IMMUTABLE_CACHE_MAGIC, + IMMUTABLE_CACHE_VERSION, + self, + "immutable data cache", + )?; std::fs::write(path, data)?; Ok(()) } diff --git a/src/cache/mod.rs b/src/cache/mod.rs index ceed40e..5ae04a2 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -8,6 +8,7 @@ pub mod snapshot; mod storage_keys; #[cfg(feature = "protocols")] mod tick_snapshot; +mod versioned; pub use binary_state::{load_binary_state, save_binary_state}; pub use metadata::{ @@ -50,7 +51,7 @@ use alloy_primitives::{Address, B256, Bytes, I256, Log, TxKind, U256, keccak256} use alloy_provider::{Provider, network::AnyNetwork}; use alloy_rpc_types_eth::TransactionRequest; use alloy_sol_types::{SolCall, SolValue, sol}; -use anyhow::{Result, anyhow}; +use anyhow::{Context as _, Result, anyhow}; use foundry_fork_db::{BlockchainDb, SharedBackend, cache::BlockchainDbMeta}; use revm::{ Context, ExecuteCommitEvm, ExecuteEvm, InspectEvm, MainBuilder, MainContext, @@ -67,8 +68,8 @@ use crate::errors::{SimError, SimulationError, SimulationResult}; use crate::freshness::SlotChange; use crate::inspector::TransferInspector; use crate::state_update::{ - AccountChange, AccountPatch, PurgeRecord, PurgeScope, SkippedBalanceDelta, SkippedDelta, - SkippedMask, SlotDelta, StateDiff, StateUpdate, + AccountChange, AccountPatch, PurgeRecord, PurgeScope, SkippedAccountPatch, SkippedBalanceDelta, + SkippedDelta, SkippedMask, SlotDelta, StateDiff, StateUpdate, }; use bytecode::BytecodeCache; @@ -183,6 +184,10 @@ fn write_slot_into( } } +fn account_patch_is_empty(patch: &AccountPatch) -> bool { + patch.balance.is_none() && patch.nonce.is_none() && patch.code.is_none() +} + static CACHE_SPEED_MODE: AtomicU8 = AtomicU8::new(CacheSpeedMode::Slow as u8); /// Runtime tuning profile for cache-side batch storage fetches. @@ -358,7 +363,8 @@ where /// Set how much EVM shared memory to pre-allocate per simulation context. /// - /// Defaults to [`SharedMemoryCapacity::Fixed`]`(64_000)` (today's behavior). + /// Defaults to [`SharedMemoryCapacity::Fixed`] with `64 * 1024` bytes + /// (65,536 bytes). /// Use `Fixed(n)` to pin a size, or [`SharedMemoryCapacity::Auto`] to size it /// from the chain state loaded at [`build`](Self::build) time (e.g. a bincode /// state file supplied via [`cache_config`](Self::cache_config)). See @@ -390,10 +396,10 @@ type InspectorCacheEvm<'a, INSP> = revm::MainnetEvm< >; /// Default initial capacity for the EVM shared-memory (working-memory) buffer. -/// 64 kB, chosen from profiling a state-heavy workload (16x the revm default of -/// 4 kB) so simulations rarely reallocate. Exposed for tuning via +/// 64 KiB (65,536 bytes), chosen from profiling a state-heavy workload (16x the +/// revm default of 4 KiB) so simulations rarely reallocate. Exposed for tuning via /// [`SharedMemoryCapacity`]. -const DEFAULT_SHARED_MEMORY_CAPACITY: usize = 64_000; +const DEFAULT_SHARED_MEMORY_CAPACITY: usize = 64 * 1024; /// How much EVM shared memory (per-context working memory) to pre-allocate for /// simulations. @@ -405,11 +411,12 @@ const DEFAULT_SHARED_MEMORY_CAPACITY: usize = 64_000; /// this much memory per overlay, so general users may want a smaller `Fixed` size, /// while state-heavy users can raise it or let it auto-size from the loaded state. /// -/// The default is `Fixed(64_000)` (today's behavior). Configure it on +/// The default is `Fixed(64 * 1024)` (65,536 bytes). Configure it on /// [`EvmCacheBuilder::shared_memory_capacity`]. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SharedMemoryCapacity { - /// Pre-allocate exactly this many bytes. The [`Default`] is `Fixed(64_000)`. + /// Pre-allocate exactly this many bytes. The [`Default`] is + /// `Fixed(64 * 1024)`. Fixed(usize), /// Size the buffer from the amount of chain state loaded into the cache at /// construction (e.g. from a bincode state file via @@ -429,7 +436,8 @@ impl Default for SharedMemoryCapacity { } impl SharedMemoryCapacity { - /// Floor for [`Auto`](Self::Auto) (and the default fixed size): 64 kB. + /// Floor for [`Auto`](Self::Auto) (and the default fixed size): 64 KiB + /// (65,536 bytes). pub const MIN_AUTO: usize = DEFAULT_SHARED_MEMORY_CAPACITY; /// Ceiling for [`Auto`](Self::Auto): 4 MiB. A simulation that needs more than /// this still works — revm grows the buffer past it on demand. @@ -1147,56 +1155,59 @@ impl EvmCache { /// /// Call this after loading AMMs and running simulations to speed up subsequent runs. /// The cache is also automatically flushed when the EvmCache is dropped. - pub fn flush(&self) { + pub fn flush(&self) -> Result<()> { if let Some(cfg) = &self.cache_config { // Save EVM state to binary cache (bincode format) let binary_path = cfg.binary_state_cache_path(); - binary_state::save_binary_state(&self.blockchain_db, &binary_path); + binary_state::save_binary_state(&self.blockchain_db, &binary_path) + .with_context(|| format!("failed to save binary state cache to {binary_path:?}"))?; // Save bytecode cache let bytecode_path = cfg.bytecode_cache_path(); let mut bytecode_cache = BytecodeCache::load(&bytecode_path).unwrap_or_default(); bytecode_cache.merge_from_db(&self.blockchain_db); - if let Err(e) = bytecode_cache.save(&bytecode_path) { - warn!(error = %e, "Failed to save bytecode cache"); - } else { - debug!( - count = bytecode_cache.contracts.len(), - path = ?bytecode_path, - "Updated bytecode cache (binary format)" - ); - } + bytecode_cache + .save(&bytecode_path) + .with_context(|| format!("failed to save bytecode cache to {bytecode_path:?}"))?; + debug!( + count = bytecode_cache.contracts.len(), + path = ?bytecode_path, + "Updated bytecode cache (binary format)" + ); // Save the immutable data cache let immutable_path = cfg.immutable_cache_path(); - if let Err(e) = self.immutable_cache.save(&immutable_path) { - warn!(error = %e, "Failed to save immutable data cache"); - } else { - debug!( - token_decimals = self.immutable_cache.token_decimals.len(), - v2_pools = self.immutable_cache.v2_pools.len(), - v3_pools = self.immutable_cache.v3_pools.len(), - balancer_pools = self.immutable_cache.balancer_pools.len(), - path = ?immutable_path, - "Updated immutable data cache" - ); - } + self.immutable_cache + .save(&immutable_path) + .with_context(|| { + format!("failed to save immutable data cache to {immutable_path:?}") + })?; + debug!( + token_decimals = self.immutable_cache.token_decimals.len(), + v2_pools = self.immutable_cache.v2_pools.len(), + v3_pools = self.immutable_cache.v3_pools.len(), + balancer_pools = self.immutable_cache.balancer_pools.len(), + path = ?immutable_path, + "Updated immutable data cache" + ); // Save the V3 tick snapshot cache (needed for liquidity validation) #[cfg(feature = "protocols")] { let tick_snapshot_path = cfg.tick_snapshot_cache_path(); - if let Err(e) = self.tick_snapshot_cache.save(&tick_snapshot_path) { - warn!(error = %e, "Failed to save V3 tick snapshot cache"); - } else { - debug!( - snapshots = self.tick_snapshot_cache.len(), - path = ?tick_snapshot_path, - "Updated V3 tick snapshot cache" - ); - } + self.tick_snapshot_cache + .save(&tick_snapshot_path) + .with_context(|| { + format!("failed to save V3 tick snapshot cache to {tick_snapshot_path:?}") + })?; + debug!( + snapshots = self.tick_snapshot_cache.len(), + path = ?tick_snapshot_path, + "Updated V3 tick snapshot cache" + ); } } + Ok(()) } /// Get the cache configuration, if any. @@ -1208,46 +1219,73 @@ impl EvmCache { self.cache_config.as_ref() } - /// Get a reference to the underlying [`BlockchainDb`] (the layer-2 backend - /// store of accounts, storage, and bytecodes). + /// Run a synchronous direct mutation against the underlying [`BlockchainDb`] + /// and invalidate the memoized snapshot base afterwards. + /// + /// This is the preferred escape hatch for unavoidable layer-2 map writes such + /// as `accounts().write().insert(...)` or `storage().write().insert(...)`. + /// The closure still bypasses the CacheDB overlay and the normal write funnel, + /// so use higher-level mutators when they can express the change. Unlike + /// [`unchecked_blockchain_db`](Self::unchecked_blockchain_db), this wrapper + /// keeps the copy-on-write snapshot base honest automatically after in-place + /// overwrites whose map cardinality does not change. + pub fn with_blockchain_db_mut(&mut self, f: impl FnOnce(&BlockchainDb) -> R) -> R { + let result = f(&self.blockchain_db); + self.invalidate_base(); + result + } + + /// Get an unchecked reference to the underlying [`BlockchainDb`] (the layer-2 + /// backend store of accounts, storage, and bytecodes). /// /// This exposes an internal store and bypasses the cache's two-layer - /// consistency model: reads here see only the backend layer, not the - /// CacheDB overlay, and any writes performed through it skip the overlay. - /// Prefer the higher-level accessors; use with care. + /// consistency model: reads here see only the backend layer, not the CacheDB + /// overlay, and any writes performed through it skip the overlay. Prefer + /// higher-level accessors or [`with_blockchain_db_mut`](Self::with_blockchain_db_mut) + /// for direct synchronous writes. /// /// # Snapshot base - /// Writing layer 2 directly through this handle also bypasses the memoized - /// copy-on-write snapshot base (Pillar A): an **in-place value overwrite at an - /// unchanged slot count** is invisible to the [`create_snapshot`](Self::create_snapshot) - /// growth scan (which is count/absence-based — the lazily-fetched backend only - /// ever *appends*, so that is sufficient for the supported write paths), and a - /// later `create_snapshot` may reuse a stale base. After a direct layer-2 write - /// through this handle, call + /// Writing layer 2 directly through this unchecked handle also bypasses the + /// memoized copy-on-write snapshot base (Pillar A). The next + /// [`create_snapshot`](Self::create_snapshot) only performs a count/absence + /// growth scan over layer 2, which catches lazy RPC-populated accounts/slots + /// because that path only appends at a fixed block. It does **not** catch + /// direct in-place changes where cardinality is unchanged: overwriting an + /// existing storage slot, or changing an existing account's info/code/balance + /// without adding a new account, can leave a stale snapshot base. After such a + /// direct write, call /// [`invalidate_snapshot_base`](Self::invalidate_snapshot_base) (or re-pin via /// [`set_block`](Self::set_block)) before the next snapshot. Writes via the /// crate's own mutators (`inject_storage_batch`, `apply_update`, the `inject_*` /// helpers, the purges) keep the base honest automatically. - pub fn blockchain_db(&self) -> &BlockchainDb { + pub fn unchecked_blockchain_db(&self) -> &BlockchainDb { &self.blockchain_db } - /// Get a reference to the underlying [`SharedBackend`] (the lazy RPC-backed - /// fetcher shared across clones). + /// Get an unchecked reference to the underlying [`SharedBackend`] (the lazy + /// RPC-backed fetcher shared across clones). /// - /// This exposes an internal and bypasses the cache's two-layer consistency + /// This exposes an internal handle and bypasses the cache's two-layer consistency /// model: it reads/fetches directly without consulting the CacheDB overlay. /// Prefer the higher-level accessors; use with care. /// /// # Snapshot base - /// `SharedBackend::insert_or_update_storage` / `insert_or_update_address` rewrite - /// layer-2 entries **in place**, which (unlike the append-only lazy fetch) can - /// leave the memoized copy-on-write snapshot base stale at an unchanged slot - /// count. After such a direct write, call + /// Lazy RPC fetches through this backend only append missing accounts/slots at + /// the pinned block, so the snapshot growth scan catches them without an + /// explicit invalidation. Direct `SharedBackend::insert_or_update_storage` / + /// `insert_or_update_address` calls are different: they enqueue a background + /// handler request that can rewrite layer-2 entries **in place**, leaving the + /// memoized copy-on-write base stale at an unchanged slot/account count. + /// + /// If you use those helpers directly, first synchronize with the backend + /// handler by reading back the updated account/slot through `SharedBackend` + /// (for example via `basic_ref` / `storage_ref`), then call /// [`invalidate_snapshot_base`](Self::invalidate_snapshot_base) before the next - /// [`create_snapshot`](Self::create_snapshot). The lazy RPC fetch path needs no - /// such call (it only ever appends, which the snapshot growth scan catches). - pub fn backend(&self) -> &SharedBackend { + /// [`create_snapshot`](Self::create_snapshot). Calling + /// `invalidate_snapshot_base` immediately after `insert_or_update_*` is not, by + /// itself, a guarantee that the queued update has been applied before the next + /// snapshot. + pub fn unchecked_backend(&self) -> &SharedBackend { &self.backend } @@ -1376,28 +1414,23 @@ impl EvmCache { /// layers (no RPC), apply each `Some` patch field (recomputing the code hash /// when `code` is set), then write through with the same layer policy. /// Records an [`AccountChange`] with `Some((old, new))` only for fields - /// that changed. + /// that changed. If the account is cold (absent from both layers), apply + /// nothing and surface a [`SkippedAccountPatch`] in + /// `diff.skipped_accounts`. + /// - [`StateUpdate::AccountUpsert`] — same patch semantics, but intentionally + /// materializes a cold/default account when absent from both layers. /// - [`StateUpdate::Purge`] — dispatch to the matching purge layer logic and /// record a [`PurgeRecord`]. /// /// # Warning — relative updates can be skipped /// - /// A relative [`SlotDelta`](StateUpdate::SlotDelta) / - /// [`BalanceDelta`](StateUpdate::BalanceDelta) targeting a **cold** address is - /// *dropped, not applied* (applying it against an unknown base would corrupt - /// state). Because a skip produces no change, it is invisible to the - /// changes-only [`StateDiff::is_empty`] / [`StateDiff::len`] success check, so - /// after applying relative updates the caller **must** inspect - /// [`StateDiff::has_skipped`] (or `diff.skipped` / `diff.skipped_balances`) and - /// fetch+seed the cold target — a silently-dropped balance update can break - /// conservation. - /// - /// # Warning — cold absolute `Account` patches - /// - /// A partial absolute [`StateUpdate::Account`] patch on an address absent from - /// both layers writes default nonce/code through the backend as authoritative, - /// masking a real RPC fetch. Fetch+seed the account first, or use - /// [`StateUpdate::BalanceDelta`] for relative native-balance tracking. + /// A cold-aware update targeting a **cold** address is *dropped, not applied* + /// unless it is an explicit [`StateUpdate::AccountUpsert`]. Because a skip + /// produces no change, it is invisible to the changes-only + /// [`StateDiff::is_empty`] / [`StateDiff::len`] success check, so after + /// applying cold-aware updates the caller **must** inspect + /// [`StateDiff::has_skipped`] (or the `skipped_*` fields) and fetch+seed the + /// cold target. /// /// ```no_run /// # use alloy_primitives::{Address, U256}; @@ -1489,7 +1522,17 @@ impl EvmCache { } } StateUpdate::Account { address, patch } => { - if let Some(change) = self.apply_account_patch(*address, patch) { + match self.apply_account_patch(*address, patch, false) { + Ok(Some(change)) => diff.accounts.push(change), + Ok(None) => {} + Err(skipped) => diff.skipped_accounts.push(skipped), + } + } + StateUpdate::AccountUpsert { address, patch } => { + if let Some(change) = self + .apply_account_patch(*address, patch, true) + .expect("AccountUpsert never skips cold account patches") + { diff.accounts.push(change); } } @@ -1872,10 +1915,22 @@ impl EvmCache { &mut self, address: Address, patch: &AccountPatch, - ) -> Option { - // 1. Current info from the cached layers only (overlay ▸ backend ▸ - // default). No RPC: apply is a write, not a fetch. - let mut info = self.loaded_account_info(address).unwrap_or_default(); + allow_cold_upsert: bool, + ) -> std::result::Result, SkippedAccountPatch> { + // 1. Current info from the cached layers only (overlay ▸ backend). No RPC: + // apply is a write, not a fetch. A partial patch on a cold account is + // skipped unless the caller explicitly chose AccountUpsert. + let mut info = match self.loaded_account_info(address) { + Some(info) => info, + None if account_patch_is_empty(patch) => return Ok(None), + None if allow_cold_upsert => AccountInfo::default(), + None => { + return Err(SkippedAccountPatch { + address, + patch: patch.clone(), + }); + } + }; let old_balance = info.balance; let old_nonce = info.nonce; @@ -1906,14 +1961,14 @@ impl EvmCache { code_hash: (old_code_hash != info.code_hash).then_some((old_code_hash, info.code_hash)), }; if change.balance.is_none() && change.nonce.is_none() && change.code_hash.is_none() { - return None; + return Ok(None); } // 4. Write-through, mirroring the slot policy: backend always; overlay // only if an overlay account already exists (do not materialize one). self.write_account_info_through(address, info); - Some(change) + Ok(Some(change)) } /// Dispatch a [`PurgeScope`] to the matching layer logic (§5.3), returning a @@ -2283,14 +2338,23 @@ impl EvmCache { /// /// The crate's own mutators keep the base honest automatically. This is the /// **escape-hatch re-honest hook**: call it after writing layer 2 directly - /// through [`blockchain_db`](Self::blockchain_db) or - /// [`backend`](Self::backend) — those bypass the write funnel, and an in-place - /// value overwrite at an unchanged slot count is invisible to the snapshot - /// growth scan (it is count/absence-based, which suffices for the append-only - /// lazy-fetch path but not for an out-of-band overwrite). Calling this before - /// the next snapshot guarantees it reflects the direct write rather than a - /// stale memoized value. Over-invalidation is always safe (Decision D2); the - /// only cost is one full base rebuild on the next snapshot. + /// through [`unchecked_blockchain_db`](Self::unchecked_blockchain_db) or + /// [`unchecked_backend`](Self::unchecked_backend) — those bypass the write + /// funnel, and in-place changes at unchanged cardinality are invisible to the + /// snapshot growth scan. + /// That includes overwriting an existing storage slot and changing an existing + /// account's info/code/balance without adding a new account. Lazy RPC-populated + /// data does not need this call because it only appends accounts/slots, which + /// the growth scan catches. + /// + /// When using `SharedBackend::insert_or_update_*` through + /// [`unchecked_backend`](Self::unchecked_backend), remember those helpers only + /// enqueue a background update. Synchronize/read back the update through + /// `SharedBackend` before the next snapshot; `invalidate_snapshot_base` alone + /// is not a backend-handler synchronization point. Once the direct write is + /// present, calling this before the next snapshot guarantees it reflects that + /// write rather than a stale memoized value. Over-invalidation is always safe + /// (Decision D2); the only cost is one full base rebuild on the next snapshot. pub fn invalidate_snapshot_base(&mut self) { self.invalidate_base(); } @@ -2337,7 +2401,7 @@ impl EvmCache { // only add a new account (caught by the absence check) or a new slot (caught // by the count check). An in-place value overwrite at unchanged length is // invisible here; the controlled writers therefore call `mark_base_dirty` - // explicitly, and a direct out-of-band write via `blockchain_db()`/`backend()` + // explicitly, and a direct out-of-band write via `unchecked_blockchain_db()`/`unchecked_backend()` // must call `invalidate_snapshot_base`. If a future foundry-fork-db bump makes // the lazy path overwrite-in-place, this scan must gain a value/version check. { @@ -2598,18 +2662,24 @@ impl EvmCache { /// To prevent the EVM block context from silently diverging from the pinned /// block, when `block` is a concrete `BlockId::Number(Number(n))` this also /// updates `block_number` (the `NUMBER` opcode) to `n`. For tag-based block - /// ids (`latest`, `pending`, hashes, etc.) the height is not statically known, - /// so `block_number` is left unchanged. - /// - /// `basefee` (the `BASEFEE` opcode) is **not** refreshed here because deriving - /// it requires fetching the block header, which this synchronous method cannot - /// do. Callers that change blocks should refresh it via - /// [`set_block_context`](Self::set_block_context) (e.g. after fetching the new - /// header). Prefer [`repin_to_block`](Self::repin_to_block) when re-pinning to + /// ids (`latest`, `pending`, hashes, etc.) and `None`, the height is not + /// statically known, so `block_number` is cleared. + /// + /// `basefee` (the `BASEFEE` opcode) is **cleared on every block change** and + /// on every non-concrete tag/hash/`None` pin call because deriving it requires + /// fetching the block header, which this synchronous method cannot do. Callers + /// that change blocks should refresh it via + /// [`set_block_context`](Self::set_block_context) after fetching the new + /// header. Prefer [`repin_to_block`](Self::repin_to_block) when re-pinning to /// a concrete height, since it keeps `block_number` and the pinned block in /// lockstep. pub fn set_block(&mut self, block: Option) { - if self.block != block { + let changed = self.block != block; + let concrete_number = match block { + Some(BlockId::Number(BlockNumberOrTag::Number(n))) => Some(n), + _ => None, + }; + if changed { self.block = block; // Re-pinning replaces layer 2 wholesale (state at a new block): the // memoized base must be rebuilt from scratch on the next snapshot. @@ -2617,14 +2687,16 @@ impl EvmCache { if let Some(block_id) = block { let _ = self.backend.set_pinned_block(block_id); *self.batch_block_id.lock().unwrap() = block_id; - // Keep the EVM `NUMBER` opcode aligned with the pinned block so the - // two cannot silently diverge. Only a concrete height is meaningful; - // tags (latest/pending/hash) leave `block_number` untouched. - if let BlockId::Number(BlockNumberOrTag::Number(n)) = block_id { - self.block_number = Some(n); - } } } + if changed || concrete_number.is_none() { + self.basefee = None; + } + + // Keep the EVM `NUMBER` opcode aligned with the pin. Only a concrete + // height is meaningful; tags, hashes, and no explicit pin clear it so a + // stale number from an earlier concrete block cannot leak into simulation. + self.block_number = concrete_number; } /// Get the block that RPC fetches are currently pinned to. @@ -2656,9 +2728,10 @@ impl EvmCache { /// Get the block number used for EVM simulations (the `NUMBER` opcode). /// - /// Fetched from the pinned block's header at construction and kept in - /// lockstep with the pin by [`set_block`](Self::set_block) / - /// [`repin_to_block`](Self::repin_to_block). `None` means revm falls back + /// Fetched from the pinned block's header at construction. Concrete-number + /// pins set it via [`set_block`](Self::set_block) / + /// [`repin_to_block`](Self::repin_to_block); tag/hash/`None` pins clear it + /// because their height is not statically known. `None` means revm falls back /// to `0`, which can steer contracts that branch on `block.number` down a /// different code path. Override directly via /// [`set_block_context`](Self::set_block_context). @@ -2669,10 +2742,12 @@ impl EvmCache { /// Get the base fee per gas used for EVM simulations (the `BASEFEE` opcode). /// /// Fetched from the pinned block's header at construction. `None` means - /// revm falls back to `0`. Unlike `block_number` this is **not** refreshed - /// by [`set_block`](Self::set_block); refresh it with - /// [`set_block_context`](Self::set_block_context) after fetching a new - /// header if `BASEFEE` accuracy matters. + /// revm falls back to `0`. This is cleared by [`set_block`](Self::set_block) + /// / [`repin_to_block`](Self::repin_to_block) when the pin changes, and by + /// non-concrete tag/hash/`None` pin calls because those can drift without a + /// concrete number in the API. Refresh it with + /// [`set_block_context`](Self::set_block_context) after fetching a new header + /// if `BASEFEE` accuracy matters. pub fn basefee(&self) -> Option { self.basefee } @@ -2717,16 +2792,12 @@ impl EvmCache { /// /// Updates the SharedBackend pinned block, the batch fetcher block, and the /// EVM block context (`NUMBER` opcode) in lockstep. The current `basefee` is - /// preserved; callers should refresh it via - /// [`set_block_context`](Self::set_block_context) after fetching the new + /// cleared because it cannot be refreshed synchronously; callers should set it + /// via [`set_block_context`](Self::set_block_context) after fetching the new /// block header if `BASEFEE` accuracy matters. pub fn repin_to_block(&mut self, block_number: u64) { let old_block = self.block; - // `set_block` already updates `block_number` for a concrete height; the - // explicit `set_block_context` below preserves `basefee` and keeps the - // re-pin atomic and self-documenting. self.set_block(Some(BlockId::Number(block_number.into()))); - self.set_block_context(Some(block_number), self.basefee); if let Some(BlockId::Number(BlockNumberOrTag::Number(old_num))) = old_block { let drift = block_number.saturating_sub(old_num); @@ -3719,8 +3790,9 @@ impl EvmCache { /// reverted. Unlike /// [`simulate_with_transfer_tracking`](Self::simulate_with_transfer_tracking), /// this measures deltas via pre/post balance reads (not transfer-event - /// inspection) and the returned - /// [`access_list`](CallSimulationResult::access_list) is always empty. + /// inspection). The returned [`access_list`](CallSimulationResult::access_list) + /// includes the accounts and slots touched by the pre/post `balanceOf` reads + /// and the simulated call. /// /// # Errors /// Returns an error if building the tx env fails, if a pre/post @@ -3768,11 +3840,13 @@ impl EvmCache { token_deltas.insert(*token, I256::from_raw(post) - I256::from_raw(pre)); } - Ok((gas_used, token_deltas, logs, output)) + let access_list = extract_access_list(&evm.journaled_state.state); + + Ok((gas_used, token_deltas, logs, output, access_list)) })(); match result { - Ok((gas_used, token_deltas, logs, output)) => { + Ok((gas_used, token_deltas, logs, output, access_list)) => { if commit { evm.commit_inner(); } else { @@ -3783,7 +3857,7 @@ impl EvmCache { gas_used, token_deltas, logs, - access_list: AccessList::default(), + access_list, output, }) } @@ -4320,7 +4394,9 @@ impl Drop for EvmCache { fn drop(&mut self) { if self.cache_config.is_some() { debug!("Flushing EVM cache on drop"); - self.flush(); + if let Err(e) = self.flush() { + warn!(error = %e, "Failed to flush EVM cache on drop"); + } } } } @@ -4352,7 +4428,7 @@ mod shared_memory_capacity_tests { #[test] fn default_is_fixed_64k() { - assert_eq!(Cap::default(), Cap::Fixed(64_000)); + assert_eq!(Cap::default(), Cap::Fixed(64 * 1024)); } #[test] @@ -4365,7 +4441,7 @@ mod shared_memory_capacity_tests { fn auto_floors_clamps_and_scales() { // Nothing / little loaded → floor. assert_eq!(Cap::Auto.resolve(0), Cap::MIN_AUTO); - assert_eq!(Cap::Auto.resolve(1_000), Cap::MIN_AUTO); // 16 KB < 64 KB floor + assert_eq!(Cap::Auto.resolve(1_000), Cap::MIN_AUTO); // 16 KiB < 64 KiB floor // Linear region (16 bytes/slot). assert_eq!(Cap::Auto.resolve(10_000), 160_000); assert_eq!(Cap::Auto.resolve(100_000), 1_600_000); @@ -4863,6 +4939,126 @@ mod core_tests { assert_eq!(cache.basefee(), None); } + #[test] + fn set_block_latest_clears_stale_block_context() { + use alloy_provider::RootProvider; + use alloy_rpc_client::RpcClient; + use alloy_transport::mock::Asserter; + + let asserter = Asserter::new(); + let client = RpcClient::mocked(asserter); + let provider = RootProvider::::new(client); + + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + + let mut cache = rt.block_on(EvmCache::new(Arc::new(provider), None)); + cache.set_block_context(Some(148_252_680), Some(50)); + + cache.set_block(Some(BlockId::latest())); + + assert_eq!( + cache.block_number(), + None, + "tag pins must not retain a stale NUMBER context" + ); + assert_eq!( + cache.basefee(), + None, + "set_block cannot refresh BASEFEE synchronously, so it must clear stale values" + ); + } + + #[test] + fn set_block_none_clears_stale_context_even_when_pin_unchanged() { + use alloy_provider::RootProvider; + use alloy_rpc_client::RpcClient; + use alloy_transport::mock::Asserter; + + let asserter = Asserter::new(); + let client = RpcClient::mocked(asserter); + let provider = RootProvider::::new(client); + + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + + let mut cache = rt.block_on(EvmCache::new(Arc::new(provider), None)); + cache.set_block_context(Some(148_252_680), Some(50)); + + cache.set_block(None); + + assert_eq!( + cache.block_number(), + None, + "None pins must not retain a stale NUMBER context" + ); + assert_eq!( + cache.basefee(), + None, + "None pins can drift like tags, so stale BASEFEE must be cleared" + ); + } + + #[test] + fn set_block_number_sets_number_and_clears_stale_basefee() { + use alloy_provider::RootProvider; + use alloy_rpc_client::RpcClient; + use alloy_transport::mock::Asserter; + + let asserter = Asserter::new(); + let client = RpcClient::mocked(asserter); + let provider = RootProvider::::new(client); + + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + + let mut cache = rt.block_on(EvmCache::new(Arc::new(provider), None)); + cache.set_block_context(Some(100), Some(50)); + + cache.set_block(Some(BlockId::Number(BlockNumberOrTag::Number(200)))); + + assert_eq!(cache.block_number(), Some(200)); + assert_eq!( + cache.basefee(), + None, + "set_block cannot refresh BASEFEE synchronously, so it must clear stale values" + ); + } + + #[test] + fn repin_to_block_clears_stale_basefee() { + use alloy_provider::RootProvider; + use alloy_rpc_client::RpcClient; + use alloy_transport::mock::Asserter; + + let asserter = Asserter::new(); + let client = RpcClient::mocked(asserter); + let provider = RootProvider::::new(client); + + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + + let mut cache = rt.block_on(EvmCache::new(Arc::new(provider), None)); + cache.set_block_context(Some(100), Some(50)); + + cache.repin_to_block(200); + + assert_eq!(cache.block_number(), Some(200)); + assert_eq!( + cache.basefee(), + None, + "repin_to_block must not carry stale BASEFEE across blocks" + ); + } + #[test] fn test_build_evm_applies_block_context() { use alloy_provider::RootProvider; @@ -4918,8 +5114,8 @@ mod core_tests { let block_num = Some(148_252_680u64); let basefee_val = Some(50u64); let child = EvmCache::from_backend( - parent.backend().clone(), - parent.blockchain_db().clone(), + parent.unchecked_backend().clone(), + parent.unchecked_blockchain_db().clone(), parent.block(), 42161, block_num, diff --git a/src/cache/tick_snapshot.rs b/src/cache/tick_snapshot.rs index e5f6f5a..145f7b2 100644 --- a/src/cache/tick_snapshot.rs +++ b/src/cache/tick_snapshot.rs @@ -12,7 +12,11 @@ use std::path::Path; use alloy_primitives::{Address, U256}; use anyhow::Result; use serde::{Deserialize, Serialize}; -use tracing::warn; + +use super::versioned; + +const TICK_SNAPSHOT_CACHE_MAGIC: &[u8; 8] = b"EFCTICK\0"; +const TICK_SNAPSHOT_CACHE_VERSION: u32 = 1; /// Per-tick liquidity state for a UniswapV3-style concentrated-liquidity pool. /// @@ -151,15 +155,16 @@ pub struct V3TickSnapshotCache { impl V3TickSnapshotCache { /// Load tick snapshot cache from disk (binary format). /// - /// Returns `None` if `path` cannot be read or its contents fail to decode as - /// bincode for this type; a decode failure is logged at `warn` level and - /// treated as a cache miss. The format has no version header, so a file from - /// an incompatible build also yields `None`. + /// Returns `None` if `path` cannot be read, fails the magic/version check, or + /// fails to decode as bincode for this type. pub fn load(path: &Path) -> Option { let data = std::fs::read(path).ok()?; - bincode::deserialize(&data) - .inspect_err(|e| warn!("Failed to parse V3 tick snapshot cache (bincode): {}", e)) - .ok() + versioned::decode( + &data, + TICK_SNAPSHOT_CACHE_MAGIC, + TICK_SNAPSHOT_CACHE_VERSION, + "V3 tick snapshot cache", + ) } /// Save tick snapshot cache to disk (binary format). @@ -175,7 +180,12 @@ impl V3TickSnapshotCache { if let Some(parent) = path.parent() { std::fs::create_dir_all(parent)?; } - let data = bincode::serialize(self)?; + let data = versioned::encode( + TICK_SNAPSHOT_CACHE_MAGIC, + TICK_SNAPSHOT_CACHE_VERSION, + self, + "V3 tick snapshot cache", + )?; std::fs::write(path, data)?; Ok(()) } diff --git a/src/cache/versioned.rs b/src/cache/versioned.rs new file mode 100644 index 0000000..0ed9681 --- /dev/null +++ b/src/cache/versioned.rs @@ -0,0 +1,66 @@ +use serde::{Serialize, de::DeserializeOwned}; +use tracing::warn; + +use anyhow::{Context as _, Result}; + +const VERSION_BYTES: usize = 4; + +pub(crate) fn encode( + magic: &[u8; 8], + version: u32, + value: &T, + label: &'static str, +) -> Result> { + let payload = + bincode::serialize(value).with_context(|| format!("failed to serialize {label}"))?; + let mut data = Vec::with_capacity(magic.len() + VERSION_BYTES + payload.len()); + data.extend_from_slice(magic); + data.extend_from_slice(&version.to_le_bytes()); + data.extend_from_slice(&payload); + Ok(data) +} + +pub(crate) fn decode( + data: &[u8], + magic: &[u8; 8], + version: u32, + label: &'static str, +) -> Option { + let header_len = magic.len() + VERSION_BYTES; + if data.len() < header_len { + warn!( + cache = label, + bytes = data.len(), + "Cache file is missing version header; treating as cache miss" + ); + return None; + } + + if &data[..magic.len()] != magic { + warn!( + cache = label, + "Cache file has unrecognized magic header; treating as cache miss" + ); + return None; + } + + let version_start = magic.len(); + let found_version = u32::from_le_bytes( + data[version_start..header_len] + .try_into() + .expect("version slice length is fixed"), + ); + if found_version != version { + warn!( + cache = label, + expected = version, + found = found_version, + "Cache file version mismatch; treating as cache miss" + ); + return None; + } + + bincode::deserialize(&data[header_len..]) + .inspect_err(|e| warn!(cache = label, error = %e, "Failed to parse cache payload")) + .ok() +} diff --git a/src/lib.rs b/src/lib.rs index 15406b8..f138977 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -128,6 +128,6 @@ pub use freshness::{ SpeculativeSim, Validation, Validity, WallClock, }; pub use state_update::{ - AccountChange, AccountPatch, PurgeRecord, PurgeScope, SkippedBalanceDelta, SkippedDelta, - SkippedMask, SlotDelta, StateDiff, StateUpdate, + AccountChange, AccountPatch, PurgeRecord, PurgeScope, SkippedAccountPatch, SkippedBalanceDelta, + SkippedDelta, SkippedMask, SlotDelta, StateDiff, StateUpdate, }; diff --git a/src/prefetch_registry.rs b/src/prefetch_registry.rs index ba5ab86..04fa1ab 100644 --- a/src/prefetch_registry.rs +++ b/src/prefetch_registry.rs @@ -16,6 +16,7 @@ use std::collections::{HashMap, HashSet}; use std::path::Path; use alloy_primitives::{Address, U256}; +use anyhow::{Context as _, Result}; use serde::{Deserialize, Serialize}; use tracing::{debug, info, warn}; @@ -80,36 +81,27 @@ impl PrefetchRegistry { /// Persist the registry to `path` in bincode format, creating parent /// directories as needed. /// - /// This is best-effort: I/O and serialization failures (unwritable parent - /// directory, failed write, or a serialization error) are logged at `warn` - /// and swallowed rather than returned, so a save failure leaves stale or - /// missing on-disk data that [`load`](Self::load) will silently treat as an - /// empty registry on the next cycle. - pub fn save(&self, path: &Path) { - if let Some(parent) = path.parent() - && let Err(e) = std::fs::create_dir_all(parent) - { - warn!(error = %e, "Failed to create prefetch registry directory"); - return; - } - match bincode::serialize(self) { - Ok(data) => { - if let Err(e) = std::fs::write(path, data) { - warn!(error = %e, "Failed to persist prefetch registry"); - } else { - let total_slots: usize = - self.phases.values().map(|al| al.slots.len()).sum::() - + self - .keyed_phases - .values() - .flat_map(|m| m.values()) - .map(|al| al.slots.len()) - .sum::(); - debug!(total_slots, "Saved prefetch registry"); - } - } - Err(e) => warn!(error = %e, "Failed to serialize prefetch registry"), + /// Returns an error if the parent directory cannot be created, serialization + /// fails, or the write fails. + pub fn save(&self, path: &Path) -> Result<()> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).with_context(|| { + format!("failed to create prefetch registry directory {parent:?}") + })?; } + let data = bincode::serialize(self).context("failed to serialize prefetch registry")?; + std::fs::write(path, data) + .with_context(|| format!("failed to persist prefetch registry to {path:?}"))?; + + let total_slots: usize = self.phases.values().map(|al| al.slots.len()).sum::() + + self + .keyed_phases + .values() + .flat_map(|m| m.values()) + .map(|al| al.slots.len()) + .sum::(); + debug!(total_slots, "Saved prefetch registry"); + Ok(()) } /// Record the aggregated access list for `phase`, **overwriting** any access @@ -352,7 +344,7 @@ mod tests { sal.slots.insert((key, U256::from(99))); registry.record_keyed("per_target", key, sal); - registry.save(&path); + registry.save(&path).expect("save registry"); let loaded = PrefetchRegistry::load(&path); assert_eq!(loaded.phases.len(), 1); @@ -374,6 +366,26 @@ mod tests { let _ = std::fs::remove_dir(&dir); } + #[test] + fn save_reports_write_failures() { + let dir = std::env::temp_dir().join("evm_fork_cache_test_prefetch_registry_write_error"); + let _ = std::fs::remove_dir_all(&dir); + let _ = std::fs::remove_file(&dir); + std::fs::write(&dir, b"not a directory").expect("create file path conflict"); + + let registry = PrefetchRegistry::default(); + let path = dir.join("registry.bin"); + let err = registry + .save(&path) + .expect_err("save must report write failure"); + assert!( + err.to_string().contains("directory") || err.to_string().contains("Not a directory"), + "unexpected error: {err:#}" + ); + + let _ = std::fs::remove_file(&dir); + } + #[test] fn test_load_missing_file_returns_default() { let path = std::path::Path::new("/tmp/nonexistent_prefetch_registry.bin"); diff --git a/src/state_update.rs b/src/state_update.rs index 165bff5..2641023 100644 --- a/src/state_update.rs +++ b/src/state_update.rs @@ -15,23 +15,31 @@ //! - [`StateUpdate::Slot`] — set a single storage slot, authoritative across //! both cache layers. //! - [`StateUpdate::Account`] — apply a partial [`AccountPatch`] -//! (`balance`/`nonce`/`code`, each optional). +//! (`balance`/`nonce`/`code`, each optional) to an already-known account. +//! - [`StateUpdate::AccountUpsert`] — intentionally materialize a cold account +//! from a partial [`AccountPatch`]. //! - [`StateUpdate::Purge`] — drop cached state at a [`PurgeScope`] so the next //! read re-fetches. //! //! # The dual-layer write-through policy //! -//! [`apply_update`](crate::cache::EvmCache::apply_update) applies a `Slot` or -//! `Account` write-through with one consistent rule: the BlockchainDb backend -//! (layer 2) is written **always**; the CacheDB overlay (layer 1) is written -//! **only if an overlay account already exists** for the address. A new overlay -//! account is never materialized for a slot/account write — the read path falls -//! through to the backend for an absent overlay entry, so a backend-only write -//! is authoritative, and materializing an overlay entry would pollute layer 1 -//! and could shadow later RPC reads. (This mirrors the established +//! [`apply_update`](crate::cache::EvmCache::apply_update) applies `Slot` writes +//! through with one consistent rule: the BlockchainDb backend (layer 2) is +//! written **always**; the CacheDB overlay (layer 1) is written **only if an +//! overlay account already exists** for the address. A new overlay account is +//! never materialized for a slot write — the read path falls through to the +//! backend for an absent overlay entry, so a backend-only write is authoritative, +//! and materializing an overlay entry would pollute layer 1 and could shadow +//! later RPC reads. (This mirrors the established //! [`inject_storage_batch_fresh`](crate::cache::EvmCache::inject_storage_batch_fresh) //! semantics.) //! +//! `Account` patches follow the same overlay-if-present write-through policy +//! once the account is already present in either layer. If the account is absent +//! from **both** layers, the patch is skipped and surfaced in +//! [`StateDiff::skipped_accounts`]. Use [`StateUpdate::AccountUpsert`] when the +//! caller intentionally wants to materialize a cold/default account. +//! //! # The output //! //! Every apply returns a [`StateDiff`] of the changes it actually made: the @@ -86,21 +94,13 @@ //! Because a cold-skipped relative update produces **no** change, it is invisible //! to the natural [`StateDiff::is_empty`] / [`StateDiff::len`] success check (those //! are changes-only). A caller applying relative updates **must** therefore check -//! [`StateDiff::has_skipped`] (or inspect [`skipped`](StateDiff::skipped) / -//! [`skipped_balances`](StateDiff::skipped_balances)) — a cold target was dropped, -//! not applied, and a silently-dropped balance update can break conservation. -//! [`StateDiff::is_fully_applied`] and [`StateDiff::skipped_len`] are the -//! companions. -//! -//! # Warning — cold absolute `Account` patches -//! -//! A *partial* absolute [`StateUpdate::Account`] patch (e.g. balance-only) on an -//! address absent from **both** cache layers writes default nonce/code through the -//! shared backend as authoritative, pre-empting a real RPC fetch. Fetch+seed the -//! account first, or prefer [`StateUpdate::BalanceDelta`] for relative -//! native-balance tracking. See the warnings on -//! [`apply_update`](crate::cache::EvmCache::apply_update), -//! [`StateUpdate::Account`], and [`AccountPatch`]. +//! [`StateDiff::has_skipped`] (or inspect [`skipped`](StateDiff::skipped), +//! [`skipped_balances`](StateDiff::skipped_balances), +//! [`skipped_masks`](StateDiff::skipped_masks), or +//! [`skipped_accounts`](StateDiff::skipped_accounts)) — a cold target was +//! dropped, not applied, and a silently-dropped balance/account update can break +//! conservation. [`StateDiff::is_fully_applied`] and +//! [`StateDiff::skipped_len`] are the companions. //! //! # Boundary — events are Phase 4 //! @@ -246,21 +246,32 @@ pub enum StateUpdate { /// The bits to write (only the bits selected by `mask` are applied). value: U256, }, - /// Patch an account's balance/nonce/code (partial — see [`AccountPatch`]). + /// Patch an already-known account's balance/nonce/code (partial — see + /// [`AccountPatch`]). /// - /// # Warning - /// - /// A partial absolute patch (e.g. balance-only) on an address absent from - /// **both** cache layers writes default nonce/code through the shared backend - /// as authoritative, pre-empting a real RPC fetch. Fetch+seed the account - /// first, or use [`StateUpdate::BalanceDelta`] for relative native-balance - /// tracking. + /// Cold-aware: if the account is absent from **both** layers, the patch is not + /// applied and is surfaced in [`StateDiff::skipped_accounts`] as a + /// [`SkippedAccountPatch`]. Use [`StateUpdate::AccountUpsert`] when + /// materializing a cold/default account is intentional. Account { /// Account to patch. address: Address, /// The partial mutation: each `Some` field overwrites, `None` leaves it. patch: AccountPatch, }, + /// Apply an [`AccountPatch`], materializing a cold account when needed. + /// + /// This is the explicit escape hatch for callers that really do want a + /// default account to become authoritative in the backend (for example a + /// synthetic test account). Normal event-derived account patches should use + /// [`StateUpdate::Account`] so a cold account is skipped instead of masking a + /// future RPC fetch. + AccountUpsert { + /// Account to patch or create. + address: Address, + /// The partial mutation: each `Some` field overwrites, `None` leaves it. + patch: AccountPatch, + }, /// Purge cached state for `address` at `scope`; the next read re-fetches. Purge { /// Account whose cached state is purged. @@ -337,6 +348,16 @@ impl StateUpdate { Self::Account { address, patch } } + /// Construct a [`StateUpdate::AccountUpsert`] from a prebuilt + /// [`AccountPatch`]. + /// + /// Use this only when materializing an account absent from both layers is the + /// desired behavior. For normal patches to known accounts, use + /// [`account`](Self::account). + pub fn account_upsert(address: Address, patch: AccountPatch) -> Self { + Self::AccountUpsert { address, patch } + } + /// Construct a [`StateUpdate::Purge`] for `address` at `scope`. pub fn purge(address: Address, scope: PurgeScope) -> Self { Self::Purge { address, scope } @@ -360,11 +381,10 @@ impl StateUpdate { /// # Warning /// /// Applying an absolute patch with [`StateUpdate::Account`] on an address absent -/// from **both** cache layers writes default values for the un-patched fields -/// (e.g. nonce `0`, empty code) through the shared backend as authoritative, -/// masking a later RPC fetch of the real on-chain account. Fetch+seed the account -/// first, or use [`StateUpdate::BalanceDelta`] for relative native-balance -/// tracking. +/// from **both** cache layers is skipped and surfaced in +/// [`StateDiff::skipped_accounts`]. Use [`StateUpdate::AccountUpsert`] only when +/// default values for un-patched fields (e.g. nonce `0`, empty code) should become +/// authoritative in the backend. /// /// ``` /// use alloy_primitives::{Bytes, U256}; @@ -448,18 +468,19 @@ pub enum PurgeScope { /// changes are recorded, so a no-op write yields a [`Default`] (empty) diff. /// /// The struct is `#[non_exhaustive]`: it has grown fields pre-1.0 -/// ([`skipped`](Self::skipped), [`skipped_balances`](Self::skipped_balances)) and -/// may grow more. Construct it via [`Default`] + field assignment, never an -/// exhaustive struct literal. +/// ([`skipped`](Self::skipped), [`skipped_balances`](Self::skipped_balances), +/// [`skipped_masks`](Self::skipped_masks), and +/// [`skipped_accounts`](Self::skipped_accounts)) and may grow more. Construct it +/// via [`Default`] + field assignment, never an exhaustive struct literal. /// /// # Checking for skips /// /// [`is_empty`](Self::is_empty) / [`len`](Self::len) are **changes-only**, so a -/// cold-skipped relative update ([`SlotDelta`](StateUpdate::SlotDelta) / -/// [`BalanceDelta`](StateUpdate::BalanceDelta)) is invisible to them. After -/// applying relative updates, check [`has_skipped`](Self::has_skipped) (or -/// inspect [`skipped`](Self::skipped) / [`skipped_balances`](Self::skipped_balances)) -/// — a cold target was dropped, not applied. +/// cold-skipped update ([`SlotDelta`](StateUpdate::SlotDelta) / +/// [`BalanceDelta`](StateUpdate::BalanceDelta) / [`Account`](StateUpdate::Account)) +/// is invisible to them. After applying cold-aware updates, check +/// [`has_skipped`](Self::has_skipped) (or inspect the `skipped_*` fields) — a cold +/// target was dropped, not applied. #[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[non_exhaustive] pub struct StateDiff { @@ -484,6 +505,10 @@ pub struct StateDiff { /// bits could not be preserved. Like [`skipped`](Self::skipped) this is /// informational metadata, not a change. pub skipped_masks: Vec, + /// Account patches ([`StateUpdate::Account`]) that were **not** applied + /// because the account was absent from both layers. Like + /// [`skipped`](Self::skipped) this is informational metadata, not a change. + pub skipped_accounts: Vec, } impl StateDiff { @@ -515,12 +540,16 @@ impl StateDiff { !self.skipped.is_empty() || !self.skipped_balances.is_empty() || !self.skipped_masks.is_empty() + || !self.skipped_accounts.is_empty() } /// Total number of skipped relative/masked updates (`skipped` + - /// `skipped_balances` + `skipped_masks`). + /// `skipped_balances` + `skipped_masks` + `skipped_accounts`). pub fn skipped_len(&self) -> usize { - self.skipped.len() + self.skipped_balances.len() + self.skipped_masks.len() + self.skipped.len() + + self.skipped_balances.len() + + self.skipped_masks.len() + + self.skipped_accounts.len() } /// Whether every relative update in the apply was applied (none skipped). @@ -535,7 +564,8 @@ impl StateDiff { /// Used by [`apply_updates`](crate::cache::EvmCache::apply_updates) to merge /// per-update diffs; the concatenation preserves order, so two writes to the /// same slot record their `old → new` history in sequence. The `skipped`, - /// `skipped_balances`, and `skipped_masks` metadata are concatenated too. + /// `skipped_balances`, `skipped_masks`, and `skipped_accounts` metadata are + /// concatenated too. pub fn merge(&mut self, other: StateDiff) { self.slots.extend(other.slots); self.accounts.extend(other.accounts); @@ -543,6 +573,7 @@ impl StateDiff { self.skipped.extend(other.skipped); self.skipped_balances.extend(other.skipped_balances); self.skipped_masks.extend(other.skipped_masks); + self.skipped_accounts.extend(other.skipped_accounts); } } @@ -642,6 +673,22 @@ pub struct SkippedMask { pub value: U256, } +/// An account patch ([`StateUpdate::Account`]) that could not be applied because +/// the account is absent from **both** cache layers. +/// +/// A partial patch against a cold account is skipped rather than applied against +/// [`AccountInfo::default`](revm::state::AccountInfo::default), because default +/// nonce/code would become authoritative and mask a later RPC fetch. It is +/// surfaced here so the caller can fetch+seed the account and retry, or opt in to +/// materialization with [`StateUpdate::AccountUpsert`]. +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct SkippedAccountPatch { + /// Account whose patch was skipped. + pub address: Address, + /// The patch that was not applied. + pub patch: AccountPatch, +} + #[cfg(test)] mod tests { use super::*; @@ -688,6 +735,13 @@ mod tests { patch: AccountPatch::default().balance(U256::from(9)), } ); + assert_eq!( + StateUpdate::account_upsert(a, AccountPatch::default().balance(U256::from(9))), + StateUpdate::AccountUpsert { + address: a, + patch: AccountPatch::default().balance(U256::from(9)), + } + ); assert_eq!( StateUpdate::purge(a, PurgeScope::Account), StateUpdate::Purge { diff --git a/tests/cache_state.rs b/tests/cache_state.rs index cdbb636..8c55b88 100644 --- a/tests/cache_state.rs +++ b/tests/cache_state.rs @@ -8,13 +8,13 @@ mod common; -use alloy_primitives::{Address, Bytes, I256, U256}; -use alloy_sol_types::SolValue; +use alloy_primitives::{Address, B256, Bytes, I256, U256, keccak256}; +use alloy_sol_types::{SolCall, SolValue}; use anyhow::{Context, Result}; use revm::state::{AccountInfo, Bytecode}; use common::{ - MOCK_ERC20_BALANCE_SLOT, balance_of, install_default_account, install_mock_erc20, + MOCK_ERC20_BALANCE_SLOT, MockERC20, balance_of, install_default_account, install_mock_erc20, mock_erc20_creation_code, mock_erc20_runtime, setup_cache, transfer, }; use evm_fork_cache::cache::TxConfig; @@ -123,6 +123,75 @@ async fn simulation_reports_balance_deltas() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread")] +async fn balance_delta_simulation_reports_access_list() -> Result<()> { + let mut cache = setup_cache().await?; + let token = Address::repeat_byte(0x44); + let owner = Address::repeat_byte(0x55); + let recipient = Address::repeat_byte(0x66); + + install_default_account(&mut cache, Address::ZERO); + install_default_account(&mut cache, owner); + install_default_account(&mut cache, recipient); + install_mock_erc20(&mut cache, token); + + let balance_slot = U256::from(MOCK_ERC20_BALANCE_SLOT); + cache.insert_mapping_storage_slot(token, balance_slot, owner, U256::from(1_000u64))?; + cache.insert_mapping_storage_slot(token, balance_slot, recipient, U256::ZERO)?; + + let transfer_call = MockERC20::transferCall { + to: recipient, + amount: U256::from(250u64), + }; + let result = cache.simulate_call_with_balance_deltas( + owner, + token, + Bytes::from(transfer_call.abi_encode()), + owner, + [token], + false, + )?; + + assert_eq!( + result.token_deltas.get(&token), + Some(&-I256::from_raw(U256::from(250u64))) + ); + + let owner_balance_slot = B256::from(U256::from_be_bytes( + keccak256((owner, balance_slot).abi_encode()).0, + )); + let recipient_balance_slot = B256::from(U256::from_be_bytes( + keccak256((recipient, balance_slot).abi_encode()).0, + )); + let token_item = result + .access_list + .0 + .iter() + .find(|item| item.address == token) + .expect("access list includes the token account"); + assert!( + token_item.storage_keys.contains(&owner_balance_slot), + "access list includes owner's balance slot" + ); + assert!( + token_item.storage_keys.contains(&recipient_balance_slot), + "access list includes recipient's balance slot" + ); + + assert_eq!( + balance_of(&mut cache, token, owner)?, + U256::from(1_000u64), + "non-committing simulation must not change owner balance" + ); + assert_eq!( + balance_of(&mut cache, token, recipient)?, + U256::ZERO, + "non-committing simulation must not change recipient balance" + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread")] async fn set_erc20_balance_with_slot_scan_finds_balance_slot() -> Result<()> { let mut cache = setup_cache().await?; @@ -235,7 +304,7 @@ async fn two_layer_cache_staleness_requires_full_purge() -> Result<()> { // Clearing ONLY the backend leaves the overlay serving stale data. { - let mut storage = cache.blockchain_db().storage().write(); + let mut storage = cache.unchecked_blockchain_db().storage().write(); storage.remove(&token); } assert_eq!( diff --git a/tests/cow_snapshot.rs b/tests/cow_snapshot.rs index 697e7f1..892e154 100644 --- a/tests/cow_snapshot.rs +++ b/tests/cow_snapshot.rs @@ -219,7 +219,7 @@ async fn cow_snapshot_matches_deep_clone_through_mutations() -> Result<()> { // `BlockchainDb` from inside foundry-fork-db, bypassing our write funnel): // a brand-new account+slot, and a NEW slot on the existing `pool`. { - let bdb = cache.blockchain_db(); + let bdb = cache.unchecked_blockchain_db(); bdb.storage() .write() .entry(pool3) @@ -275,7 +275,7 @@ async fn cow_snapshot_matches_deep_clone_through_mutations() -> Result<()> { } /// Escape-hatch re-honest hook (adversarial-review finding). A direct, out-of-band -/// layer-2 write through `blockchain_db()` that overwrites an existing slot at an +/// layer-2 write through `unchecked_blockchain_db()` that overwrites an existing slot at an /// unchanged slot count is the one mutation the count-based growth scan cannot see, /// so the memoized base can go stale. `invalidate_snapshot_base()` must restore /// read-equivalence with the deep-clone reference. @@ -290,7 +290,7 @@ async fn invalidate_snapshot_base_rehonest_after_escape_hatch_write() -> Result< // Out-of-band overwrite at unchanged length (bypasses the write funnel). { - let bdb = cache.blockchain_db(); + let bdb = cache.unchecked_blockchain_db(); bdb.storage() .write() .entry(pool) @@ -311,6 +311,195 @@ async fn invalidate_snapshot_base_rehonest_after_escape_hatch_write() -> Result< Ok(()) } +/// Escape-hatch re-honest hook for account-map overwrites. A direct update of an +/// existing layer-2 account's balance/code at an unchanged account count is also +/// invisible to the count/absence growth scan, so callers must invalidate the +/// memoized base after the direct write lands. +#[tokio::test(flavor = "multi_thread")] +async fn invalidate_snapshot_base_rehonest_after_existing_account_write() -> Result<()> { + let mut cache = setup_cache().await?; + let account = Address::repeat_byte(0xA1); + let code_v1 = Bytecode::new_raw(Bytes::from(vec![0x60u8, 0x01])); + let code_v2 = Bytecode::new_raw(Bytes::from(vec![0x60u8, 0x02, 0x60, 0x03])); + let h1 = code_v1.hash_slow(); + let h2 = code_v2.hash_slow(); + assert_ne!(h1, h2); + + let original = AccountInfo { + balance: U256::from(111u64), + nonce: 1, + code_hash: h1, + code: Some(code_v1.clone()), + account_id: None, + }; + let updated = AccountInfo { + balance: U256::from(222u64), + nonce: 2, + code_hash: h2, + code: Some(code_v2.clone()), + account_id: None, + }; + + { + let bdb = cache.unchecked_blockchain_db(); + bdb.accounts().write().insert(account, original.clone()); + } + let warm = cache.create_snapshot(); // memoize the base with `original`. + let mut ov_warm = EvmOverlay::new(Arc::clone(&warm), None); + let warm_info = ov_warm + .basic(account) + .expect("warm basic") + .expect("warm account"); + assert_eq!(warm_info.balance, original.balance); + assert_eq!(warm_info.nonce, original.nonce); + assert_eq!(warm_info.code_hash, h1); + assert_eq!( + ov_warm + .code_by_hash(h1) + .expect("warm code") + .original_bytes(), + code_v1.original_bytes() + ); + + // Out-of-band account overwrite at unchanged account count (bypasses the + // write funnel and is not detectable by the growth scan). + { + let bdb = cache.unchecked_blockchain_db(); + let mut accounts = bdb.accounts().write(); + assert!( + accounts.contains_key(&account), + "test must update an existing account" + ); + let len_before = accounts.len(); + accounts.insert(account, updated.clone()); + assert_eq!( + accounts.len(), + len_before, + "test must keep the account count unchanged" + ); + } + + cache.invalidate_snapshot_base(); + let cow = cache.create_snapshot(); + let deep = cache.create_snapshot_deep_clone(); + let mut ov_cow = EvmOverlay::new(Arc::clone(&cow), None); + let mut ov_deep = EvmOverlay::new(Arc::clone(&deep), None); + + let cow_basic = ov_cow.basic(account).expect("cow basic"); + let deep_basic = ov_deep.basic(account).expect("deep basic"); + assert!( + account_eq(&cow_basic, &deep_basic), + "invalidate_snapshot_base must re-honest the base after an out-of-band account write: cow={cow_basic:?} deep={deep_basic:?}" + ); + let cow_info = cow_basic.expect("updated cow account"); + assert_eq!(cow_info.balance, updated.balance); + assert_eq!(cow_info.nonce, updated.nonce); + assert_eq!(cow_info.code_hash, h2); + assert_eq!( + ov_cow.code_by_hash(h2).expect("cow h2").original_bytes(), + ov_deep.code_by_hash(h2).expect("deep h2").original_bytes(), + "updated code hash must match the deep clone" + ); + assert_eq!( + ov_cow.code_by_hash(h2).expect("cow h2").original_bytes(), + code_v2.original_bytes() + ); + assert!( + ov_deep.code_by_hash(h1).expect("deep h1").is_empty(), + "sanity: deep clone drops the unreferenced old hash" + ); + assert_eq!( + ov_cow.code_by_hash(h1).expect("cow h1").original_bytes(), + ov_deep.code_by_hash(h1).expect("deep h1").original_bytes(), + "the old code hash must not linger after invalidation" + ); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn with_blockchain_db_mut_rehonest_after_storage_overwrite() -> Result<()> { + let mut cache = setup_cache().await?; + let pool = Address::repeat_byte(0x78); + let slot = U256::from(0u64); + + cache.inject_storage_batch(&[(pool, slot, U256::from(111u64))]); + let _warm = cache.create_snapshot(); + + cache.with_blockchain_db_mut(|bdb| { + bdb.storage() + .write() + .entry(pool) + .or_default() + .insert(slot, U256::from(222u64)); + }); + + let cow = cache.create_snapshot(); + let deep = cache.create_snapshot_deep_clone(); + assert_eq!( + cow.storage_value(pool, slot), + deep.storage_value(pool, slot), + "with_blockchain_db_mut must invalidate the COW base after storage writes" + ); + assert_eq!(cow.storage_value(pool, slot), Some(U256::from(222u64))); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn with_blockchain_db_mut_rehonest_after_account_overwrite() -> Result<()> { + let mut cache = setup_cache().await?; + let account = Address::repeat_byte(0xA2); + let code_v1 = Bytecode::new_raw(Bytes::from(vec![0x60u8, 0x01])); + let code_v2 = Bytecode::new_raw(Bytes::from(vec![0x60u8, 0x02])); + let h1 = code_v1.hash_slow(); + let h2 = code_v2.hash_slow(); + let original = AccountInfo { + balance: U256::from(111u64), + nonce: 1, + code_hash: h1, + code: Some(code_v1), + account_id: None, + }; + let updated = AccountInfo { + balance: U256::from(222u64), + nonce: 2, + code_hash: h2, + code: Some(code_v2.clone()), + account_id: None, + }; + + cache.with_blockchain_db_mut(|bdb| { + bdb.accounts().write().insert(account, original); + }); + let _warm = cache.create_snapshot(); + + cache.with_blockchain_db_mut(|bdb| { + let mut accounts = bdb.accounts().write(); + let len_before = accounts.len(); + accounts.insert(account, updated.clone()); + assert_eq!(accounts.len(), len_before); + }); + + let cow = cache.create_snapshot(); + let deep = cache.create_snapshot_deep_clone(); + let mut ov_cow = EvmOverlay::new(Arc::clone(&cow), None); + let mut ov_deep = EvmOverlay::new(Arc::clone(&deep), None); + let cow_basic = ov_cow.basic(account).expect("cow basic"); + let deep_basic = ov_deep.basic(account).expect("deep basic"); + assert!( + account_eq(&cow_basic, &deep_basic), + "with_blockchain_db_mut must invalidate the COW base after account writes: cow={cow_basic:?} deep={deep_basic:?}" + ); + assert_eq!( + cow_basic.expect("updated cow account").balance, + updated.balance + ); + assert_eq!( + ov_cow.code_by_hash(h2).expect("cow h2").original_bytes(), + code_v2.original_bytes() + ); + Ok(()) +} + /// Regression (review finding P2): the COW partial rebuild must not leave a stale /// `code_by_hash` entry when a base account is recoded or purged. Warm the base /// with a code-bearing account, recode it in layer 2, dirty it via a controlled @@ -330,7 +519,7 @@ async fn cow_code_index_matches_deep_clone_after_base_account_recoded() -> Resul // Seed a code-bearing account (code_v1) directly into the cold base (layer 2), // then re-honest the memoized base. let put_account = |cache: &EvmCache, code: &Bytecode, hash| { - cache.blockchain_db().accounts().write().insert( + cache.unchecked_blockchain_db().accounts().write().insert( contract, AccountInfo { balance: U256::from(1u64), diff --git a/tests/event_pipeline.rs b/tests/event_pipeline.rs index d18d224..bc6b2be 100644 --- a/tests/event_pipeline.rs +++ b/tests/event_pipeline.rs @@ -9,7 +9,7 @@ //! //! Layering vocabulary mirrors `tests/state_update.rs`: //! - **layer 1 / overlay** = the CacheDB overlay (`db_mut().cache.accounts`). -//! - **layer 2 / backend** = the BlockchainDb backend (`blockchain_db()`). +//! - **layer 2 / backend** = the BlockchainDb backend (`unchecked_blockchain_db()`). mod common; @@ -41,7 +41,7 @@ fn mapping_slot(owner: Address, mapping_slot: u64) -> U256 { /// Value of a slot in the BlockchainDb backend (layer 2) only. fn backend_slot(cache: &EvmCache, addr: Address, slot: U256) -> Option { cache - .blockchain_db() + .unchecked_blockchain_db() .storage() .read() .get(&addr) diff --git a/tests/freshness.rs b/tests/freshness.rs index 73607d0..554d96d 100644 --- a/tests/freshness.rs +++ b/tests/freshness.rs @@ -248,7 +248,7 @@ async fn purge_account_drops_account_and_storage_from_both_layers() -> Result<() ); // Account gone from the backend accounts map. { - let accounts = cache.blockchain_db().accounts().read(); + let accounts = cache.unchecked_blockchain_db().accounts().read(); assert!(!accounts.contains_key(&token), "backend account removed"); } @@ -1572,8 +1572,8 @@ async fn run_unverified_without_fetcher() -> Result<()> { // A `from_backend` cache exposes no fetcher (no provider captured). let base = cache_with_balance(token, owner, U256::from(1000)).await?; let mut cache = EvmCache::from_backend( - base.backend().clone(), - base.blockchain_db().clone(), + base.unchecked_backend().clone(), + base.unchecked_blockchain_db().clone(), None, base.chain_id(), None, diff --git a/tests/serialization_roundtrip.rs b/tests/serialization_roundtrip.rs index 71850cf..27bdc32 100644 --- a/tests/serialization_roundtrip.rs +++ b/tests/serialization_roundtrip.rs @@ -87,6 +87,16 @@ fn immutable_data_cache_round_trips() { let len_before = cache.len(); cache.save(&path).expect("save immutable cache"); + let bytes = std::fs::read(&path).expect("read immutable cache file"); + assert!( + bytes.starts_with(b"EFCMETA\0"), + "immutable cache must carry a magic header" + ); + assert_eq!( + &bytes[8..12], + &1u32.to_le_bytes(), + "immutable cache must carry an explicit version" + ); let loaded = ImmutableDataCache::load(&path).expect("load immutable cache"); // Counts and scalar values survive the round trip. @@ -118,6 +128,20 @@ fn immutable_data_cache_round_trips() { assert_eq!(bal.last_change_block, U256::from(18_000_000u64)); } +#[test] +fn immutable_data_cache_load_legacy_raw_bincode_is_none() { + let dir = TempDir::new("immutable_legacy"); + let path = dir.path("legacy_immutable_data.bin"); + let mut cache = ImmutableDataCache::default(); + cache.set_token_decimals(Address::repeat_byte(0xA1), 6); + std::fs::write(&path, bincode::serialize(&cache).unwrap()).expect("write legacy cache"); + + assert!( + ImmutableDataCache::load(&path).is_none(), + "unversioned legacy bincode must be treated as a cache miss" + ); +} + #[test] fn immutable_data_cache_load_missing_file_is_none() { let dir = TempDir::new("immutable_missing"); @@ -180,6 +204,16 @@ mod tick_snapshots { assert_eq!(cache.len(), 1); cache.save(&path).expect("save tick cache"); + let bytes = std::fs::read(&path).expect("read tick cache file"); + assert!( + bytes.starts_with(b"EFCTICK\0"), + "tick snapshot cache must carry a magic header" + ); + assert_eq!( + &bytes[8..12], + &1u32.to_le_bytes(), + "tick snapshot cache must carry an explicit version" + ); let loaded = V3TickSnapshotCache::load(&path).expect("load tick cache"); let snap = loaded.get(pool).expect("snapshot present"); @@ -190,6 +224,24 @@ mod tick_snapshots { assert_eq!(snap.to_ticks(), ticks, "ticks survive round trip"); } + #[test] + fn v3_tick_snapshot_cache_load_legacy_raw_bincode_is_none() { + let dir = TempDir::new("v3_ticks_legacy"); + let path = dir.path("legacy_v3_tick_snapshots.bin"); + let pool = Address::repeat_byte(0x77); + let mut cache = V3TickSnapshotCache::default(); + cache.set( + pool, + V3PoolTickSnapshot::from_pool_data(&HashMap::new(), &HashMap::new(), 0, 0), + ); + std::fs::write(&path, bincode::serialize(&cache).unwrap()).expect("write legacy cache"); + + assert!( + V3TickSnapshotCache::load(&path).is_none(), + "unversioned legacy bincode must be treated as a cache miss" + ); + } + #[test] fn v3_tick_snapshot_silently_drops_unparseable_keys() { // Pin the documented behavior (KNOWN_ISSUES): a string key that does not diff --git a/tests/shared_memory_capacity.rs b/tests/shared_memory_capacity.rs index 08a9bf4..f3fc029 100644 --- a/tests/shared_memory_capacity.rs +++ b/tests/shared_memory_capacity.rs @@ -36,8 +36,8 @@ async fn default_capacity_is_fixed_64k() -> Result<()> { let cache = EvmCacheBuilder::new(mock_provider()).build().await; assert_eq!( cache.shared_memory_capacity(), - 64_000, - "the default must be Fixed(64_000)" + 65_536, + "the default must be Fixed(64 * 1024)" ); Ok(()) } @@ -54,7 +54,7 @@ async fn fixed_capacity_is_honored() -> Result<()> { #[tokio::test(flavor = "multi_thread")] async fn auto_capacity_with_no_loaded_state_falls_back_to_floor() -> Result<()> { - // No cache_config → nothing loaded → Auto resolves to the 64 KB floor. + // No cache_config → nothing loaded → Auto resolves to the 64 KiB floor. let cache = EvmCacheBuilder::new(mock_provider()) .shared_memory_capacity(SharedMemoryCapacity::Auto) .build() @@ -69,7 +69,7 @@ async fn auto_capacity_with_no_loaded_state_falls_back_to_floor() -> Result<()> /// The headline: `Auto` sizes the buffer from the chain state in a loaded bincode /// state file. A first cache persists 10 000 storage slots; a second cache built /// with `Auto` over the same `CacheConfig` loads them and pre-allocates -/// `10_000 * 16 = 160_000` bytes (vs. the 64 KB default). +/// `10_000 * 16 = 160_000` bytes (vs. the 64 KiB default). #[tokio::test(flavor = "multi_thread")] async fn auto_capacity_scales_with_loaded_binary_state() -> Result<()> { let dir = unique_cache_dir("auto"); @@ -86,7 +86,7 @@ async fn auto_capacity_scales_with_loaded_binary_state() -> Result<()> { .map(|i| (token, U256::from(i), U256::from(i + 1))) .collect(); cache.inject_storage_batch(&batch); - cache.flush(); // writes evm_state.bin + cache.flush()?; // writes evm_state.bin } // Second cache: Auto over the same config loads the 10k slots and sizes from them. @@ -104,11 +104,34 @@ async fn auto_capacity_scales_with_loaded_binary_state() -> Result<()> { // A Fixed override ignores the loaded state. let fixed = EvmCacheBuilder::new(mock_provider()) .cache_config(cfg.clone()) - .shared_memory_capacity(SharedMemoryCapacity::Fixed(64_000)) + .shared_memory_capacity(SharedMemoryCapacity::Fixed(64 * 1024)) .build() .await; - assert_eq!(fixed.shared_memory_capacity(), 64_000); + assert_eq!(fixed.shared_memory_capacity(), 65_536); let _ = std::fs::remove_dir_all(&dir); Ok(()) } + +#[tokio::test(flavor = "multi_thread")] +async fn flush_reports_unwritable_cache_paths() -> Result<()> { + let path_conflict = unique_cache_dir("flush_error"); + std::fs::write(&path_conflict, b"not a directory")?; + let cfg = CacheConfig::new(&path_conflict, 1, Default::default(), Default::default()); + let cache = EvmCacheBuilder::new(mock_provider()) + .cache_config(cfg) + .build() + .await; + + let err = cache + .flush() + .expect_err("flush must report persistence failures"); + let rendered = format!("{err:#}"); + assert!( + rendered.contains("directory") || rendered.contains("Not a directory"), + "unexpected error: {rendered}" + ); + + let _ = std::fs::remove_file(&path_conflict); + Ok(()) +} diff --git a/tests/state_update.rs b/tests/state_update.rs index 2ee024e..815758e 100644 --- a/tests/state_update.rs +++ b/tests/state_update.rs @@ -10,7 +10,7 @@ //! - **layer 1 / overlay** = the CacheDB overlay (`db_mut().cache.accounts`), //! which wins on reads. //! - **layer 2 / backend** = the BlockchainDb backend -//! (`blockchain_db().storage()` / `.accounts()`). +//! (`unchecked_blockchain_db().storage()` / `.accounts()`). mod common; @@ -22,8 +22,8 @@ use common::{ }; use evm_fork_cache::cache::EvmCache; use evm_fork_cache::{ - AccountPatch, PurgeScope, SkippedBalanceDelta, SkippedDelta, SkippedMask, SlotChange, - SlotDelta, StateDiff, StateUpdate, + AccountPatch, PurgeScope, SkippedAccountPatch, SkippedBalanceDelta, SkippedDelta, SkippedMask, + SlotChange, SlotDelta, StateDiff, StateUpdate, }; use revm::state::{AccountInfo, Bytecode}; @@ -52,7 +52,7 @@ fn overlay_slot(cache: &mut EvmCache, addr: Address, slot: U256) -> Option /// Value of a slot in the BlockchainDb backend (layer 2) only. fn backend_slot(cache: &EvmCache, addr: Address, slot: U256) -> Option { cache - .blockchain_db() + .unchecked_blockchain_db() .storage() .read() .get(&addr) @@ -87,7 +87,7 @@ fn overlay_nonce(cache: &mut EvmCache, addr: Address) -> Option { /// Backend (layer 2) balance for `addr`, if a backend account exists. fn backend_balance(cache: &EvmCache, addr: Address) -> Option { cache - .blockchain_db() + .unchecked_blockchain_db() .accounts() .read() .get(&addr) @@ -133,6 +133,13 @@ fn state_update_constructors_produce_expected_variants() { patch: AccountPatch::default().balance(U256::from(9)), } ); + assert_eq!( + StateUpdate::account_upsert(a, AccountPatch::default().balance(U256::from(9))), + StateUpdate::AccountUpsert { + address: a, + patch: AccountPatch::default().balance(U256::from(9)), + } + ); assert_eq!( StateUpdate::purge(a, PurgeScope::Account), StateUpdate::Purge { @@ -341,15 +348,46 @@ async fn apply_account_code_patch_recomputes_hash() -> Result<()> { } #[tokio::test] -async fn apply_account_patch_materializes_absent_account() -> Result<()> { - // An account absent from both layers is created (in the backend) by a patch, - // and the value is readable. +async fn apply_account_patch_on_cold_account_is_skipped_and_surfaced() -> Result<()> { + // A partial Account patch against a cold account must not materialize a + // default backend account, because that would mask the real on-chain account. let addr = Address::repeat_byte(0x88); let mut cache = setup_cache().await?; assert!(!overlay_has_account(&mut cache, addr)); assert_eq!(backend_balance(&cache, addr), None); - let diff = cache.apply_update(&StateUpdate::balance(addr, U256::from(1234))); + let patch = AccountPatch::default().balance(U256::from(1234)); + let diff = cache.apply_update(&StateUpdate::account(addr, patch.clone())); + + assert_eq!( + backend_balance(&cache, addr), + None, + "cold patch must not materialize a backend account" + ); + assert!(diff.accounts.is_empty()); + assert_eq!( + diff.skipped_accounts, + vec![SkippedAccountPatch { + address: addr, + patch + }] + ); + assert!(diff.has_skipped()); + assert_eq!(diff.skipped_len(), 1); + Ok(()) +} + +#[tokio::test] +async fn account_upsert_intentionally_materializes_absent_account() -> Result<()> { + let addr = Address::repeat_byte(0x88); + let mut cache = setup_cache().await?; + assert!(!overlay_has_account(&mut cache, addr)); + assert_eq!(backend_balance(&cache, addr), None); + + let diff = cache.apply_update(&StateUpdate::account_upsert( + addr, + AccountPatch::default().balance(U256::from(1234)), + )); assert_eq!(backend_balance(&cache, addr), Some(U256::from(1234))); assert_eq!(diff.accounts.len(), 1); @@ -357,6 +395,7 @@ async fn apply_account_patch_materializes_absent_account() -> Result<()> { diff.accounts[0].balance, Some((U256::ZERO, U256::from(1234))) ); + assert!(diff.skipped_accounts.is_empty()); Ok(()) } @@ -394,7 +433,7 @@ async fn apply_purge_account_clears_both_layers() -> Result<()> { "backend storage gone" ); { - let accounts = cache.blockchain_db().accounts().read(); + let accounts = cache.unchecked_blockchain_db().accounts().read(); assert!(!accounts.contains_key(&token), "backend account removed"); } assert_eq!(diff.purged.len(), 1); @@ -1303,7 +1342,7 @@ async fn account_patch_on_backend_only_account_does_not_materialize_overlay() -> let acct = Address::repeat_byte(0x91); let mut cache = setup_cache().await?; // Seed only the backend (the cold-prefetched, layer-2-only case). - cache.blockchain_db().accounts().write().insert( + cache.unchecked_blockchain_db().accounts().write().insert( acct, AccountInfo { balance: U256::from(100), @@ -1582,7 +1621,7 @@ async fn account_patch_normalizes_zero_code_hash_across_layers() -> Result<()> { use revm::primitives::KECCAK_EMPTY; let acct = Address::repeat_byte(0x7f); let mut cache = setup_cache().await?; - cache.blockchain_db().accounts().write().insert( + cache.unchecked_blockchain_db().accounts().write().insert( acct, AccountInfo { balance: U256::from(1), @@ -1594,7 +1633,7 @@ async fn account_patch_normalizes_zero_code_hash_across_layers() -> Result<()> { cache.apply_update(&StateUpdate::balance(acct, U256::from(2))); let backend_hash = cache - .blockchain_db() + .unchecked_blockchain_db() .accounts() .read() .get(&acct) From 149e4dda47db7f6d0fb2efca08fce01c1af6a30c Mon Sep 17 00:00:00 2001 From: Kai Aldag Date: Wed, 17 Jun 2026 14:52:52 +0100 Subject: [PATCH 26/26] Deflake into_optimistic freshness abort test --- tests/freshness.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/freshness.rs b/tests/freshness.rs index 554d96d..aa2455a 100644 --- a/tests/freshness.rs +++ b/tests/freshness.rs @@ -1185,6 +1185,12 @@ async fn run_unverified_on_fetcher_error() -> Result<()> { // T3 (part 2): into_optimistic aborts the validation task. The fetcher WOULD // queue a correction (it reports a changed value), so if the abort failed we // would observe a non-zero pending queue. We assert it stays 0. +// +// Determinism mirrors the Drop-abort test below: the validator is allowed to +// reach the synchronous fetch, but the gated fetch cannot return until after +// `into_optimistic()` has set the cancel flag. That makes the product guarantee +// precise: a cancel observed at the post-fetch checkpoint suppresses all +// side-effects, including pending corrections and re-run accounting. #[tokio::test(flavor = "multi_thread")] async fn run_into_optimistic_aborts_validation() -> Result<()> { let token = Address::repeat_byte(0x44); @@ -1192,11 +1198,12 @@ async fn run_into_optimistic_aborts_validation() -> Result<()> { let recipient = Address::repeat_byte(0x66); let mut cache = cache_with_balance(token, owner, U256::from(1000)).await?; + let gate = Gate::new(); // A CHANGED value: if the validator ran, it would queue a correction. - cache.set_storage_batch_fetcher(stub_fetcher(HashMap::from([( - (token, balance_slot_for(owner)), - U256::from(50), - )]))); + cache.set_storage_batch_fetcher(gated_tracking_fetcher( + HashMap::from([((token, balance_slot_for(owner)), U256::from(50))]), + gate.clone(), + )); let mut controller = FreshnessController::new(FreshnessRegistry::new(), AlwaysVerify); let sim = controller.run( @@ -1208,6 +1215,7 @@ async fn run_into_optimistic_aborts_validation() -> Result<()> { )], )?; let results = sim.into_optimistic(); // aborts the background validation + gate.release(); assert_eq!(results.len(), 1); // Give any (incorrectly) surviving task a chance to run, then assert no