diff --git a/.github/scripts/bench-txgen-run.sh b/.github/scripts/bench-txgen-run.sh index 500c3a97a69..58c73cd24e5 100755 --- a/.github/scripts/bench-txgen-run.sh +++ b/.github/scripts/bench-txgen-run.sh @@ -293,12 +293,18 @@ cleanup() { sleep 1 fi sudo systemctl reset-failed "$RETH_SCOPE" 2>/dev/null || true + rm -f "$JWT_SECRET" sudo chown -R "$(id -un):$(id -gn)" "$OUTPUT_DIR" 2>/dev/null || true sudo schelk recover -y --kill || true } TAIL_PID= TRACY_PID= +# reth runs as root and creates a missing JWT secret with 0600 permissions, which +# the bench client (run as this user) cannot read. Create it owned by this user, +# outside the uploaded output dir, once cleanup is set to remove it. +JWT_SECRET="$(mktemp)" trap cleanup EXIT +od -An -N32 -tx1 /dev/urandom | tr -d ' \n' > "$JWT_SECRET" sudo systemctl stop "$RETH_SCOPE" 2>/dev/null || true sudo systemctl reset-failed "$RETH_SCOPE" 2>/dev/null || true @@ -341,6 +347,7 @@ RETH_ARGS=( --ws --ws.api all --authrpc.port 8551 + --authrpc.jwtsecret "$JWT_SECRET" --disable-discovery --no-persist-peers ) @@ -689,7 +696,7 @@ if [ "$WARMUP" -gt 0 ] 2>/dev/null; then else $BENCH_NICE "$TXGEN_BENCH" send-blocks \ --engine http://127.0.0.1:8551 \ - --jwt-secret "$DATADIR/jwt.hex" \ + --jwt-secret "$JWT_SECRET" \ --input "$WARMUP_BLOCKS" \ "${TXGEN_SEND_ARGS[@]}" \ --wait-for-persistence never \ @@ -791,7 +798,7 @@ elif [ "$EXECUTION_MODE" = "rpc" ]; then TXGEN_REPLAY_ARGS=(send --rpc-url http://127.0.0.1:8545 --drain-timeout 300) BENCH_SCENARIO="rpc-replay" else - TXGEN_REPLAY_ARGS=(send-blocks --engine http://127.0.0.1:8551 --jwt-secret "$DATADIR/jwt.hex" --wait-for-persistence never) + TXGEN_REPLAY_ARGS=(send-blocks --engine http://127.0.0.1:8551 --jwt-secret "$JWT_SECRET" --wait-for-persistence never) fi # No Engine API traffic in call mode, so `latest` must not move while measuring. diff --git a/.github/workflows/bench-scheduled.yml b/.github/workflows/bench-scheduled.yml index fbf621294b5..8fb1f936952 100644 --- a/.github/workflows/bench-scheduled.yml +++ b/.github/workflows/bench-scheduled.yml @@ -293,6 +293,8 @@ jobs: contents: read actions: read timeout-minutes: 120 + outputs: + failed-step: ${{ steps.slack-failure.outputs.failed-step }} env: BENCH_RPC_URL: http://ethereum-mainnet-stable-reth-greedy-goose:8545 SCHELK_MOUNT: /reth-bench @@ -370,7 +372,7 @@ jobs: toolchain: stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 continue-on-error: true - - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 + - uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 with: version: "0.13.4" @@ -993,6 +995,7 @@ jobs: } - name: Send Slack notification (failure) + id: slack-failure if: failure() && env.BENCH_SLACK != 'never' && env.BENCH_SLACK != 'on-win' uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: @@ -1000,10 +1003,6 @@ jobs: SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }} with: script: | - const token = process.env.SLACK_BENCH_BOT_TOKEN; - const channel = process.env.SLACK_BENCH_CHANNEL; - if (!token || !channel) return; - const steps_status = [ ['installing dependencies', '${{ steps.install-deps.outcome }}'], ['checking dependencies', '${{ steps.check-deps.outcome }}'], @@ -1016,6 +1015,12 @@ jobs: ]; const failed = steps_status.find(([, o]) => o === 'failure'); const failedStep = failed ? failed[0] : 'unknown step'; + // Read by the incident-io job. + core.setOutput('failed-step', failedStep); + + const token = process.env.SLACK_BENCH_BOT_TOKEN; + const channel = process.env.SLACK_BENCH_CHANNEL; + if (!token || !channel) return; const repo = `${context.repo.owner}/${context.repo.repo}`; const jobUrl = process.env.BENCH_JOB_URL || `${context.serverUrl}/${repo}/actions/runs/${context.runId}`; @@ -1030,7 +1035,7 @@ jobs: }, { type: 'section', - text: { type: 'mrkdwn', text: `*${modeLabel} regression* failed while *${failedStep}*\ncc <@U09FARE0B9Q> <@U09FAL2UMLJ>\n<@U0ANX3AM5RR> investigate this` }, + text: { type: 'mrkdwn', text: `*${modeLabel} regression* failed while *${failedStep}*\ncc <@U09FARE0B9Q> <@U09FAL2UMLJ>` }, }, { type: 'actions', @@ -1072,7 +1077,118 @@ jobs: sudo systemctl start irqbalance cron atd 2>/dev/null || true # --------------------------------------------------------------------------- - # Job 3: Save state on success + # Job 3: Page reth on-call through incident.io on scheduled failures and resolve + # the same per-mode alert on the next successful scheduled run. Skipped runs (no + # new commits) leave the alert as it is. Skips when the secrets are not set. + # --------------------------------------------------------------------------- + incident-io: + needs: [resolve-refs, bench-scheduled] + if: "!cancelled() && github.event_name == 'schedule' && github.repository == 'paradigmxyz/reth'" + name: incident-io + runs-on: ubuntu-latest + permissions: {} + steps: + - name: Send incident.io alert event + env: + INCIDENT_IO_URL: ${{ secrets.INCIDENT_IO_RETH_CI_URL }} + INCIDENT_IO_TOKEN: ${{ secrets.INCIDENT_IO_RETH_CI_TOKEN }} + REFS_RESULT: ${{ needs.resolve-refs.result }} + BENCH_RESULT: ${{ needs.bench-scheduled.result }} + BENCH_MODE: ${{ needs.resolve-refs.outputs.mode }} + FAILED_STEP: ${{ needs.bench-scheduled.outputs.failed-step }} + IS_STALE: ${{ needs.resolve-refs.outputs.is-stale }} + BASELINE_REF: ${{ needs.resolve-refs.outputs.baseline-ref }} + FEATURE_REF: ${{ needs.resolve-refs.outputs.feature-ref }} + SHA: ${{ github.sha }} + BRANCH: ${{ github.ref_name }} + run: | + if [ "$REFS_RESULT" = "failure" ] || [ "$BENCH_RESULT" = "failure" ]; then + STATUS="firing" + elif [ "$BENCH_RESULT" = "success" ]; then + STATUS="resolved" + else + echo "Benchmark did not run (resolve-refs: $REFS_RESULT, bench-scheduled: $BENCH_RESULT); leaving the incident.io alert unchanged" + exit 0 + fi + + if [ -z "$INCIDENT_IO_URL" ] || [ -z "$INCIDENT_IO_TOKEN" ]; then + echo "::notice::INCIDENT_IO_RETH_CI_URL or INCIDENT_IO_RETH_CI_TOKEN is not set; skipping incident.io alert" + exit 0 + fi + + MODE="${BENCH_MODE:-unknown}" + case "$MODE" in + release) MODE_LABEL="Release" ;; + hourly) MODE_LABEL="Hourly" ;; + *) MODE_LABEL="Nightly" ;; + esac + DEDUP_KEY="reth:bench-$MODE" + RUN_URL="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + + if [ "$STATUS" = "resolved" ]; then + TITLE="Reth $MODE_LABEL bench recovered" + DETAIL="The scheduled $MODE bench (bench-scheduled.yml) succeeded." + elif [ "$REFS_RESULT" = "failure" ]; then + if [ "$IS_STALE" = "true" ]; then + TITLE="Reth $MODE_LABEL bench failed: nightly build is stale" + DETAIL="The scheduled $MODE bench (bench-scheduled.yml) aborted because the nightly Docker build is older than 24 hours." + else + TITLE="Reth $MODE_LABEL bench failed while resolving refs" + DETAIL="The scheduled $MODE bench (bench-scheduled.yml) failed while resolving benchmark refs." + fi + else + STEP="${FAILED_STEP:-unknown step}" + TITLE="Reth $MODE_LABEL bench failed while $STEP" + DETAIL="The scheduled $MODE bench (bench-scheduled.yml) failed while $STEP." + fi + DESCRIPTION=$(printf '%s\nBaseline: %s\nFeature: %s\nCommit: %s\nBranch: %s\nRun: %s' \ + "$DETAIL" "${BASELINE_REF:-unknown}" "${FEATURE_REF:-unknown}" "$SHA" "$BRANCH" "$RUN_URL") + + PAYLOAD=$(jq -n \ + --arg title "$TITLE" \ + --arg description "$DESCRIPTION" \ + --arg status "$STATUS" \ + --arg dedup "$DEDUP_KEY" \ + --arg run_url "$RUN_URL" \ + --arg repository "$GITHUB_REPOSITORY" \ + --arg mode "$MODE" \ + --arg failed_step "$FAILED_STEP" \ + --arg baseline_ref "$BASELINE_REF" \ + --arg feature_ref "$FEATURE_REF" \ + --arg sha "$SHA" \ + --arg branch "$BRANCH" \ + --arg run_id "$GITHUB_RUN_ID" \ + '{ + title: $title, + description: $description, + status: $status, + deduplication_key: $dedup, + source_url: $run_url, + metadata: { + team: "reth", + repository: $repository, + workflow: "bench-scheduled.yml", + mode: $mode, + failed_step: $failed_step, + baseline_ref: $baseline_ref, + feature_ref: $feature_ref, + sha: $sha, + branch: $branch, + run_id: $run_id, + run_url: $run_url + } + }') + + curl -fsS --retry 3 --retry-all-errors --max-time 30 \ + -X POST "$INCIDENT_IO_URL" \ + -H "Authorization: Bearer $INCIDENT_IO_TOKEN" \ + -H "Content-Type: application/json" \ + --data "$PAYLOAD" + echo + echo "Sent incident.io $STATUS event for $DEDUP_KEY" + + # --------------------------------------------------------------------------- + # Job 4: Save state on success # --------------------------------------------------------------------------- save-state: needs: [resolve-refs, bench-scheduled] diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 859f8fd9636..aa0e51a248f 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -1376,7 +1376,7 @@ jobs: toolchain: stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 continue-on-error: true - - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 + - uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 with: version: "0.13.4" diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index 8508940ae63..1768cbb939c 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -22,6 +22,8 @@ jobs: env: # Include the system trust store containing Aegis's CA for Bun and Node. NODE_EXTRA_CA_CERTS: /etc/ssl/certs/ca-certificates.crt + # Read by build-cargo-docs.sh, which `bun run build` also invokes. + RUST_NIGHTLY: nightly-2026-10-31 steps: - name: Secure runner uses: tempoxyz/gh-actions/actions/secure-runner@d5402286a7abdfb7232173b3228d56473feb8a66 # 2026-09-29T02-04-45Z-d5402286 @@ -47,7 +49,7 @@ jobs: - name: Install Rust nightly uses: tempoxyz/gh-actions/vendor/dtolnay/rust-toolchain@d5402286a7abdfb7232173b3228d56473feb8a66 # 2026-09-29T02-04-45Z-d5402286 with: - toolchain: nightly + toolchain: ${{ env.RUST_NIGHTLY }} - name: Build docs run: cd docs/vocs && bash scripts/build-cargo-docs.sh diff --git a/.github/workflows/check-alloy.yml b/.github/workflows/check-alloy.yml index 1343e40a74d..2b2e0eaf90e 100644 --- a/.github/workflows/check-alloy.yml +++ b/.github/workflows/check-alloy.yml @@ -42,7 +42,7 @@ jobs: with: toolchain: stable - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: cache-on-failure: true diff --git a/.github/workflows/compact.yml b/.github/workflows/compact.yml index 60c5044b539..326eff11251 100644 --- a/.github/workflows/compact.yml +++ b/.github/workflows/compact.yml @@ -47,7 +47,7 @@ jobs: with: toolchain: stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: cache-on-failure: true - name: Checkout base diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ae325d2885e..b861b6831c5 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -173,8 +173,76 @@ jobs: *Run:* *Action required:* Re-run the workflow or investigate the build failure. - - <@U0ANX3AM5RR> investigate and re-run if flaky SLACK_FOOTER: "paradigmxyz/reth ยท docker.yml" MSG_MINIMAL: true SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + + # Pages reth on-call through incident.io on failure, and resolves the same alert on the + # next successful scheduled build. Skips when the incident.io secrets are not set. + incident-io: + name: Update incident.io alert + runs-on: ubuntu-latest + needs: build + if: "!cancelled() && github.event_name == 'schedule' && github.repository == 'paradigmxyz/reth' && (needs.build.result == 'success' || needs.build.result == 'failure')" + permissions: {} + steps: + - name: Send incident.io alert event + env: + INCIDENT_IO_URL: ${{ secrets.INCIDENT_IO_RETH_CI_URL }} + INCIDENT_IO_TOKEN: ${{ secrets.INCIDENT_IO_RETH_CI_TOKEN }} + BUILD_RESULT: ${{ needs.build.result }} + SHA: ${{ github.sha }} + BRANCH: ${{ github.ref_name }} + run: | + if [ -z "$INCIDENT_IO_URL" ] || [ -z "$INCIDENT_IO_TOKEN" ]; then + echo "::notice::INCIDENT_IO_RETH_CI_URL or INCIDENT_IO_RETH_CI_TOKEN is not set; skipping incident.io alert" + exit 0 + fi + + RUN_URL="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + if [ "$BUILD_RESULT" = "success" ]; then + STATUS="resolved" + TITLE="Reth nightly Docker build recovered" + OUTCOME="succeeded" + else + STATUS="firing" + TITLE="Reth nightly Docker build failed" + OUTCOME="failed" + fi + DESCRIPTION=$(printf 'The scheduled nightly Docker build (docker.yml) %s.\nCommit: %s\nBranch: %s\nRun: %s\nRe-run the workflow if the failure is flaky; otherwise investigate the build.' \ + "$OUTCOME" "$SHA" "$BRANCH" "$RUN_URL") + + PAYLOAD=$(jq -n \ + --arg title "$TITLE" \ + --arg description "$DESCRIPTION" \ + --arg status "$STATUS" \ + --arg run_url "$RUN_URL" \ + --arg repository "$GITHUB_REPOSITORY" \ + --arg sha "$SHA" \ + --arg branch "$BRANCH" \ + --arg run_id "$GITHUB_RUN_ID" \ + '{ + title: $title, + description: $description, + status: $status, + deduplication_key: "reth:docker-nightly", + source_url: $run_url, + metadata: { + team: "reth", + repository: $repository, + workflow: "docker.yml", + mode: "nightly", + sha: $sha, + branch: $branch, + run_id: $run_id, + run_url: $run_url + } + }') + + curl -fsS --retry 3 --retry-all-errors --max-time 30 \ + -X POST "$INCIDENT_IO_URL" \ + -H "Authorization: Bearer $INCIDENT_IO_TOKEN" \ + -H "Content-Type: application/json" \ + --data "$PAYLOAD" + echo + echo "Sent incident.io $STATUS event for reth:docker-nightly" diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 6b44ae9b9e7..28474983e64 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -53,7 +53,7 @@ jobs: tool: nextest@0.9.140 checksum: true fallback: none - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: prefix-key: v1-rust-e2e-targets cache-bin: false diff --git a/.github/workflows/fetch-grafana-dashboard.yml b/.github/workflows/fetch-grafana-dashboard.yml index 4e7e24714b8..ef7fb64f865 100644 --- a/.github/workflows/fetch-grafana-dashboard.yml +++ b/.github/workflows/fetch-grafana-dashboard.yml @@ -25,7 +25,7 @@ jobs: with: persist-credentials: false - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: python-version: "3.12" diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 19ac263959b..4592b9c5ee0 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -64,7 +64,7 @@ jobs: checksum: true fallback: none - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: # Restoring the full target cache exhausts the integration runner's disk. # sccache still reuses compiled artifacts without unpacking that cache. @@ -117,7 +117,7 @@ jobs: checksum: true fallback: none - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: cache-on-failure: true - name: run era/era1/ere files integration tests diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 06d0ba32458..15ceac49757 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -47,10 +47,10 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: tempoxyz/gh-actions/vendor/dtolnay/rust-toolchain@d5402286a7abdfb7232173b3228d56473feb8a66 # 2026-09-29T02-04-45Z-d5402286 with: - toolchain: nightly + toolchain: nightly-2026-10-31 components: clippy - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: cache-on-failure: true - name: Run clippy on binaries @@ -75,10 +75,10 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: tempoxyz/gh-actions/vendor/dtolnay/rust-toolchain@d5402286a7abdfb7232173b3228d56473feb8a66 # 2026-09-29T02-04-45Z-d5402286 with: - toolchain: nightly + toolchain: nightly-2026-10-31 components: clippy - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: cache-on-failure: true - run: cargo clippy --workspace --lib --examples --tests --benches --all-features --locked @@ -113,7 +113,7 @@ jobs: checksum: true fallback: none - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: cache-on-failure: true - uses: dcarbone/install-jq-action@4fcb5062d7ce9bc4382d1a352d19ba3ba2c317c1 # v4.0.1 @@ -146,7 +146,7 @@ jobs: checksum: true fallback: none - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: cache-on-failure: true - uses: dcarbone/install-jq-action@4fcb5062d7ce9bc4382d1a352d19ba3ba2c317c1 # v4.0.1 @@ -205,7 +205,7 @@ jobs: with: toolchain: "1.96" # MSRV - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: cache-on-failure: true - run: cargo build --bin reth --workspace @@ -229,9 +229,9 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: tempoxyz/gh-actions/vendor/dtolnay/rust-toolchain@d5402286a7abdfb7232173b3228d56473feb8a66 # 2026-09-29T02-04-45Z-d5402286 with: - toolchain: nightly + toolchain: nightly-2026-10-31 - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: cache-on-failure: true - run: cargo docs --document-private-items @@ -279,9 +279,9 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: tempoxyz/gh-actions/vendor/dtolnay/rust-toolchain@d5402286a7abdfb7232173b3228d56473feb8a66 # 2026-09-29T02-04-45Z-d5402286 with: - toolchain: nightly + toolchain: nightly-2026-10-31 - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: cache-on-failure: true - name: Install cargo-udeps @@ -309,9 +309,9 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: tempoxyz/gh-actions/vendor/dtolnay/rust-toolchain@d5402286a7abdfb7232173b3228d56473feb8a66 # 2026-09-29T02-04-45Z-d5402286 with: - toolchain: nightly + toolchain: nightly-2026-10-31 - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: cache-on-failure: true - run: cargo build --bin reth --workspace diff --git a/.github/workflows/stage.yml b/.github/workflows/stage.yml index 8645fe1f390..4991029ba80 100644 --- a/.github/workflows/stage.yml +++ b/.github/workflows/stage.yml @@ -53,7 +53,7 @@ jobs: with: toolchain: stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: cache-on-failure: true - name: Build reth diff --git a/.github/workflows/sync-era.yml b/.github/workflows/sync-era.yml index 2ca18233663..28fb12875a2 100644 --- a/.github/workflows/sync-era.yml +++ b/.github/workflows/sync-era.yml @@ -50,7 +50,7 @@ jobs: with: toolchain: stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: cache-on-failure: true - name: Build ${{ matrix.chain.bin }} diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 9f50133df06..21a96695d9c 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -50,7 +50,7 @@ jobs: with: toolchain: stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: cache-on-failure: true - name: Build ${{ matrix.chain.bin }} diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index 10268dc907f..12820c01214 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -56,7 +56,7 @@ jobs: with: toolchain: stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: prefix-key: v1-rust-unit-targets cache-bin: false @@ -124,7 +124,7 @@ jobs: checksum: true fallback: none - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: cache-on-failure: true - name: Prepare tmpfs temp dir @@ -166,7 +166,7 @@ jobs: - name: Compute cache week id: cache-week run: echo "week=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT" - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: key: ${{ steps.cache-week.outputs.week }} cache-on-failure: true diff --git a/ACTIVATIONS.md b/ACTIVATIONS.md index fe1fe55ec25..73f4bf9cbc5 100644 --- a/ACTIVATIONS.md +++ b/ACTIVATIONS.md @@ -5,8 +5,8 @@ Firehose reth node serves and its effect on the Firehose block. Review the entri whose status changed at every upstream bump, and when an activation is announced between bumps. Statuses come from the `alloy-hardforks` timestamps compiled into the release (0.4.11 for -`reth-v2.9.0-fh3.1`), checked 2026-10-29. "Not scheduled" means the fork has no timestamp in the -code: it reaches the tracer only once a later release schedules it. +`reth-v2.9.0-fh3.1` and `reth-v2.10.0-fh3.1`), checked 2026-11-06. "Not scheduled" means the fork +has no timestamp in the code: it reaches the tracer only once a later release schedules it. | Fork | Mainnet | Sepolia | Hoodi | Effect on the Firehose block | |------|---------|---------|-------|------------------------------| @@ -18,4 +18,4 @@ code: it reaches the tracer only once a later release schedules it. | BPO1 | Active (2025-12-09) | Active (2025-10-21) | Active (2025-11-05) | None: blob parameters only. | | BPO2 | Active (2026-01-07) | Active (2025-10-27) | Active (2025-11-12) | None: blob parameters only. | | BPO3, BPO4, BPO5 | Not scheduled | Not scheduled | Not scheduled | None expected: blob parameters only. Re-check when scheduled. | -| Amsterdam | Not scheduled | Active (2026-10-06, block 11856337) | Not scheduled (unscheduled in v2.8.1) | Header gains `block_access_list_rlp` and `slot_number`; EIP-7708 native transfer logs; EIP-8037 state gas in call gas; EIP-2780 transactions included without a root frame are traced as an out-of-gas root call; EIP-8282 builder request system calls; EIP-8246 self-beneficiary SELFDESTRUCT no longer burns. Covered by the `amsterdam_*` prestate goldens and `firehose_live_tracing_amsterdam`. | +| Amsterdam (Glamsterdam) | Not scheduled | Active (2026-10-06, block 11856337) | Not scheduled (unscheduled in v2.8.1, no new date as of 2026-11-06) | Header gains `block_access_list_rlp` and `slot_number`; EIP-7708 native transfer logs; EIP-8037 state gas in call gas; EIP-2780 transactions included without a root frame are traced as an out-of-gas root call; EIP-8282 builder request system calls; EIP-8246 self-beneficiary SELFDESTRUCT no longer burns. Covered by the `amsterdam_*` prestate goldens and `firehose_live_tracing_amsterdam`. | diff --git a/CHANGELOG.md b/CHANGELOG.md index 9474f0a0d3f..aff0872f267 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,13 @@ This changelog covers Firehose-specific changes only. For upstream reth changes, The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). -## reth-v2.9.0-fh3.1-2-rc.1 +## reth-v2.10.0-fh3.1 -Tests the automated release tag, no Firehose changes. +### Changed + +- Update to upstream reth [v2.10.0](https://github.com/paradigmxyz/reth/releases/tag/v2.10.0). + evm2 comes from the `streamingfast/evm2` fork tag `reth-v2.10.0-sf`. No hardfork is scheduled + or changed, and the Firehose block output does not change. ## reth-v2.9.0-fh3.1-1 diff --git a/Cargo.lock b/Cargo.lock index 7ad236a8c50..493517fd97a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3424,7 +3424,7 @@ dependencies = [ [[package]] name = "ef-test-runner" -version = "2.9.0" +version = "2.10.0" dependencies = [ "clap", "ef-tests", @@ -3432,7 +3432,7 @@ dependencies = [ [[package]] name = "ef-tests" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -3603,7 +3603,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3680,7 +3680,7 @@ dependencies = [ [[package]] name = "evm2" version = "0.1.0" -source = "git+https://github.com/streamingfast/evm2?tag=reth-v2.9.0-sf#6814eff8437528a2e4eafd5b768ea2b34aa1d0c5" +source = "git+https://github.com/streamingfast/evm2?tag=reth-v2.10.0-sf#e1f625f1cdbef4b1d2e586092e8bd53540f75be0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -3720,7 +3720,7 @@ dependencies = [ [[package]] name = "evm2-inspectors" version = "0.1.0" -source = "git+https://github.com/streamingfast/evm2?tag=reth-v2.9.0-sf#6814eff8437528a2e4eafd5b768ea2b34aa1d0c5" +source = "git+https://github.com/streamingfast/evm2?tag=reth-v2.10.0-sf#e1f625f1cdbef4b1d2e586092e8bd53540f75be0" dependencies = [ "alloy-consensus", "alloy-eip2930", @@ -3743,7 +3743,7 @@ dependencies = [ [[package]] name = "evm2-jit" version = "0.1.0" -source = "git+https://github.com/streamingfast/evm2?tag=reth-v2.9.0-sf#6814eff8437528a2e4eafd5b768ea2b34aa1d0c5" +source = "git+https://github.com/streamingfast/evm2?tag=reth-v2.10.0-sf#e1f625f1cdbef4b1d2e586092e8bd53540f75be0" dependencies = [ "evm2", "evm2-jit-backend", @@ -3758,7 +3758,7 @@ dependencies = [ [[package]] name = "evm2-jit-backend" version = "0.1.0" -source = "git+https://github.com/streamingfast/evm2?tag=reth-v2.9.0-sf#6814eff8437528a2e4eafd5b768ea2b34aa1d0c5" +source = "git+https://github.com/streamingfast/evm2?tag=reth-v2.10.0-sf#e1f625f1cdbef4b1d2e586092e8bd53540f75be0" dependencies = [ "eyre", "ruint", @@ -3767,12 +3767,12 @@ dependencies = [ [[package]] name = "evm2-jit-build" version = "0.1.0" -source = "git+https://github.com/streamingfast/evm2?tag=reth-v2.9.0-sf#6814eff8437528a2e4eafd5b768ea2b34aa1d0c5" +source = "git+https://github.com/streamingfast/evm2?tag=reth-v2.10.0-sf#e1f625f1cdbef4b1d2e586092e8bd53540f75be0" [[package]] name = "evm2-jit-builtins" version = "0.1.0" -source = "git+https://github.com/streamingfast/evm2?tag=reth-v2.9.0-sf#6814eff8437528a2e4eafd5b768ea2b34aa1d0c5" +source = "git+https://github.com/streamingfast/evm2?tag=reth-v2.10.0-sf#e1f625f1cdbef4b1d2e586092e8bd53540f75be0" dependencies = [ "alloy-primitives", "evm2", @@ -3785,7 +3785,7 @@ dependencies = [ [[package]] name = "evm2-jit-codegen" version = "0.1.0" -source = "git+https://github.com/streamingfast/evm2?tag=reth-v2.9.0-sf#6814eff8437528a2e4eafd5b768ea2b34aa1d0c5" +source = "git+https://github.com/streamingfast/evm2?tag=reth-v2.10.0-sf#e1f625f1cdbef4b1d2e586092e8bd53540f75be0" dependencies = [ "alloy-primitives", "bitflags 2.13.2", @@ -3808,7 +3808,7 @@ dependencies = [ [[package]] name = "evm2-jit-context" version = "0.1.0" -source = "git+https://github.com/streamingfast/evm2?tag=reth-v2.9.0-sf#6814eff8437528a2e4eafd5b768ea2b34aa1d0c5" +source = "git+https://github.com/streamingfast/evm2?tag=reth-v2.10.0-sf#e1f625f1cdbef4b1d2e586092e8bd53540f75be0" dependencies = [ "alloy-primitives", "evm2", @@ -3818,7 +3818,7 @@ dependencies = [ [[package]] name = "evm2-jit-llvm" version = "0.1.0" -source = "git+https://github.com/streamingfast/evm2?tag=reth-v2.9.0-sf#6814eff8437528a2e4eafd5b768ea2b34aa1d0c5" +source = "git+https://github.com/streamingfast/evm2?tag=reth-v2.10.0-sf#e1f625f1cdbef4b1d2e586092e8bd53540f75be0" dependencies = [ "alloy-primitives", "cc", @@ -3831,7 +3831,7 @@ dependencies = [ [[package]] name = "evm2-jit-runtime" version = "0.1.0" -source = "git+https://github.com/streamingfast/evm2?tag=reth-v2.9.0-sf#6814eff8437528a2e4eafd5b768ea2b34aa1d0c5" +source = "git+https://github.com/streamingfast/evm2?tag=reth-v2.10.0-sf#e1f625f1cdbef4b1d2e586092e8bd53540f75be0" dependencies = [ "alloy-primitives", "crossbeam-channel", @@ -3858,7 +3858,7 @@ dependencies = [ [[package]] name = "evm2-macros" version = "0.1.0" -source = "git+https://github.com/streamingfast/evm2?tag=reth-v2.9.0-sf#6814eff8437528a2e4eafd5b768ea2b34aa1d0c5" +source = "git+https://github.com/streamingfast/evm2?tag=reth-v2.10.0-sf#e1f625f1cdbef4b1d2e586092e8bd53540f75be0" dependencies = [ "proc-macro2", "quote", @@ -4115,7 +4115,7 @@ dependencies = [ [[package]] name = "example-full-contract-state" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-primitives", "eyre", @@ -4233,7 +4233,7 @@ dependencies = [ [[package]] name = "exex-subscription" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-primitives", "eyre", @@ -5566,7 +5566,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6668,7 +6668,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7737,7 +7737,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -8118,7 +8118,7 @@ checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" [[package]] name = "reth" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-node-bindings", "alloy-primitives", @@ -8174,7 +8174,7 @@ dependencies = [ [[package]] name = "reth-basic-payload-builder" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8200,7 +8200,7 @@ dependencies = [ [[package]] name = "reth-bb" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8236,7 +8236,7 @@ dependencies = [ [[package]] name = "reth-chain-state" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8265,7 +8265,7 @@ dependencies = [ [[package]] name = "reth-chainspec" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-chains", "alloy-consensus", @@ -8284,7 +8284,7 @@ dependencies = [ [[package]] name = "reth-cli" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-genesis", "clap", @@ -8296,7 +8296,7 @@ dependencies = [ [[package]] name = "reth-cli-commands" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-chains", "alloy-consensus", @@ -8386,7 +8386,7 @@ dependencies = [ [[package]] name = "reth-cli-runner" -version = "2.9.0" +version = "2.10.0" dependencies = [ "reth-tasks", "tokio", @@ -8395,7 +8395,7 @@ dependencies = [ [[package]] name = "reth-cli-util" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8446,7 +8446,7 @@ dependencies = [ [[package]] name = "reth-config" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-primitives", "eyre", @@ -8464,7 +8464,7 @@ dependencies = [ [[package]] name = "reth-consensus" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -8477,7 +8477,7 @@ dependencies = [ [[package]] name = "reth-consensus-common" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8491,7 +8491,7 @@ dependencies = [ [[package]] name = "reth-consensus-debug-client" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8515,7 +8515,7 @@ dependencies = [ [[package]] name = "reth-db" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8549,7 +8549,7 @@ dependencies = [ [[package]] name = "reth-db-api" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8578,7 +8578,7 @@ dependencies = [ [[package]] name = "reth-db-common" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-genesis", @@ -8609,7 +8609,7 @@ dependencies = [ [[package]] name = "reth-db-models" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8625,7 +8625,7 @@ dependencies = [ [[package]] name = "reth-discv4" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8652,7 +8652,7 @@ dependencies = [ [[package]] name = "reth-discv5" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8677,7 +8677,7 @@ dependencies = [ [[package]] name = "reth-dns-discovery" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-chains", "alloy-primitives", @@ -8705,7 +8705,7 @@ dependencies = [ [[package]] name = "reth-downloaders" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8745,7 +8745,7 @@ dependencies = [ [[package]] name = "reth-dst" -version = "2.9.0" +version = "2.10.0" dependencies = [ "bedrock-assertions", "serde", @@ -8754,7 +8754,7 @@ dependencies = [ [[package]] name = "reth-e2e-test-utils" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8816,7 +8816,7 @@ dependencies = [ [[package]] name = "reth-ecies" -version = "2.9.0" +version = "2.10.0" dependencies = [ "aes", "alloy-primitives", @@ -8842,7 +8842,7 @@ dependencies = [ [[package]] name = "reth-engine-local" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8864,7 +8864,7 @@ dependencies = [ [[package]] name = "reth-engine-primitives" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8892,7 +8892,7 @@ dependencies = [ [[package]] name = "reth-engine-tree" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -8969,7 +8969,7 @@ dependencies = [ [[package]] name = "reth-engine-util" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8996,7 +8996,7 @@ dependencies = [ [[package]] name = "reth-era" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9020,7 +9020,7 @@ dependencies = [ [[package]] name = "reth-era-downloader" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-primitives", "bytes", @@ -9038,7 +9038,7 @@ dependencies = [ [[package]] name = "reth-era-utils" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9068,7 +9068,7 @@ dependencies = [ [[package]] name = "reth-errors" -version = "2.9.0" +version = "2.10.0" dependencies = [ "reth-consensus", "reth-execution-errors", @@ -9078,7 +9078,7 @@ dependencies = [ [[package]] name = "reth-eth-wire" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-chains", "alloy-consensus", @@ -9116,7 +9116,7 @@ dependencies = [ [[package]] name = "reth-eth-wire-types" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-chains", "alloy-consensus", @@ -9144,7 +9144,7 @@ dependencies = [ [[package]] name = "reth-ethereum" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-rpc-types-engine", "alloy-rpc-types-eth", @@ -9183,7 +9183,7 @@ dependencies = [ [[package]] name = "reth-ethereum-cli" -version = "2.9.0" +version = "2.10.0" dependencies = [ "clap", "eyre", @@ -9206,7 +9206,7 @@ dependencies = [ [[package]] name = "reth-ethereum-consensus" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9222,7 +9222,7 @@ dependencies = [ [[package]] name = "reth-ethereum-engine-primitives" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -9238,7 +9238,7 @@ dependencies = [ [[package]] name = "reth-ethereum-forks" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-eip2124", "alloy-hardforks 0.4.11", @@ -9251,7 +9251,7 @@ dependencies = [ [[package]] name = "reth-ethereum-payload-builder" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9279,7 +9279,7 @@ dependencies = [ [[package]] name = "reth-ethereum-primitives" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9300,7 +9300,7 @@ dependencies = [ [[package]] name = "reth-etl" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-primitives", "rayon", @@ -9310,7 +9310,7 @@ dependencies = [ [[package]] name = "reth-evm" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9338,7 +9338,7 @@ dependencies = [ [[package]] name = "reth-evm-ethereum" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9365,7 +9365,7 @@ dependencies = [ [[package]] name = "reth-execution-cache" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-primitives", "evm2", @@ -9384,7 +9384,7 @@ dependencies = [ [[package]] name = "reth-execution-errors" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -9395,7 +9395,7 @@ dependencies = [ [[package]] name = "reth-execution-types" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9416,7 +9416,7 @@ dependencies = [ [[package]] name = "reth-exex" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9462,7 +9462,7 @@ dependencies = [ [[package]] name = "reth-exex-test-utils" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-eips", "eyre", @@ -9493,7 +9493,7 @@ dependencies = [ [[package]] name = "reth-exex-types" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -9510,7 +9510,7 @@ dependencies = [ [[package]] name = "reth-firehose" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9538,7 +9538,7 @@ dependencies = [ [[package]] name = "reth-firehose-tests" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9564,7 +9564,7 @@ dependencies = [ [[package]] name = "reth-fs-util" -version = "2.9.0" +version = "2.10.0" dependencies = [ "serde", "serde_json", @@ -9573,7 +9573,7 @@ dependencies = [ [[package]] name = "reth-invalid-block-hooks" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9604,7 +9604,7 @@ dependencies = [ [[package]] name = "reth-ipc" -version = "2.9.0" +version = "2.10.0" dependencies = [ "bytes", "futures", @@ -9626,7 +9626,7 @@ dependencies = [ [[package]] name = "reth-libmdbx" -version = "2.9.0" +version = "2.10.0" dependencies = [ "bitflags 2.13.2", "byteorder", @@ -9643,7 +9643,7 @@ dependencies = [ [[package]] name = "reth-mdbx-sys" -version = "2.9.0" +version = "2.10.0" dependencies = [ "bindgen", "cc", @@ -9651,7 +9651,7 @@ dependencies = [ [[package]] name = "reth-metrics" -version = "2.9.0" +version = "2.10.0" dependencies = [ "futures", "libc", @@ -9664,7 +9664,7 @@ dependencies = [ [[package]] name = "reth-net-banlist" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-primitives", "ipnet", @@ -9672,7 +9672,7 @@ dependencies = [ [[package]] name = "reth-net-nat" -version = "2.9.0" +version = "2.10.0" dependencies = [ "futures-util", "if-addrs", @@ -9686,7 +9686,7 @@ dependencies = [ [[package]] name = "reth-network" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9757,7 +9757,7 @@ dependencies = [ [[package]] name = "reth-network-api" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9781,7 +9781,7 @@ dependencies = [ [[package]] name = "reth-network-p2p" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9805,7 +9805,7 @@ dependencies = [ [[package]] name = "reth-network-peers" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -9822,7 +9822,7 @@ dependencies = [ [[package]] name = "reth-network-types" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-eip2124", "humantime-serde", @@ -9835,7 +9835,7 @@ dependencies = [ [[package]] name = "reth-nippy-jar" -version = "2.9.0" +version = "2.10.0" dependencies = [ "anyhow", "bincode", @@ -9854,7 +9854,7 @@ dependencies = [ [[package]] name = "reth-node-api" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-rpc-types-engine", "eyre", @@ -9877,7 +9877,7 @@ dependencies = [ [[package]] name = "reth-node-builder" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9951,7 +9951,7 @@ dependencies = [ [[package]] name = "reth-node-core" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10006,7 +10006,7 @@ dependencies = [ [[package]] name = "reth-node-ethereum" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-contract", @@ -10090,7 +10090,7 @@ dependencies = [ [[package]] name = "reth-node-ethstats" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -10113,7 +10113,7 @@ dependencies = [ [[package]] name = "reth-node-events" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10136,7 +10136,7 @@ dependencies = [ [[package]] name = "reth-node-metrics" -version = "2.9.0" +version = "2.10.0" dependencies = [ "bytes", "eyre", @@ -10168,7 +10168,7 @@ dependencies = [ [[package]] name = "reth-node-types" -version = "2.9.0" +version = "2.10.0" dependencies = [ "reth-chainspec", "reth-db-api", @@ -10179,7 +10179,7 @@ dependencies = [ [[package]] name = "reth-payload-builder" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -10202,7 +10202,7 @@ dependencies = [ [[package]] name = "reth-payload-builder-primitives" -version = "2.9.0" +version = "2.10.0" dependencies = [ "pin-project", "reth-payload-primitives", @@ -10213,7 +10213,7 @@ dependencies = [ [[package]] name = "reth-payload-primitives" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10237,7 +10237,7 @@ dependencies = [ [[package]] name = "reth-payload-util" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -10246,7 +10246,7 @@ dependencies = [ [[package]] name = "reth-payload-validator" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-rpc-types-engine", @@ -10286,7 +10286,7 @@ dependencies = [ [[package]] name = "reth-provider" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -10338,7 +10338,7 @@ dependencies = [ [[package]] name = "reth-prune" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -10368,7 +10368,7 @@ dependencies = [ [[package]] name = "reth-prune-types" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-primitives", "arbitrary", @@ -10388,7 +10388,7 @@ dependencies = [ [[package]] name = "reth-rpc" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -10469,7 +10469,7 @@ dependencies = [ [[package]] name = "reth-rpc-api" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-eips", "alloy-genesis", @@ -10498,7 +10498,7 @@ dependencies = [ [[package]] name = "reth-rpc-builder" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-eips", "alloy-network", @@ -10556,7 +10556,7 @@ dependencies = [ [[package]] name = "reth-rpc-convert" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-json-rpc", @@ -10575,7 +10575,7 @@ dependencies = [ [[package]] name = "reth-rpc-e2e-tests" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-genesis", "eyre", @@ -10594,7 +10594,7 @@ dependencies = [ [[package]] name = "reth-rpc-engine-api" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10630,7 +10630,7 @@ dependencies = [ [[package]] name = "reth-rpc-eth-api" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -10674,7 +10674,7 @@ dependencies = [ [[package]] name = "reth-rpc-eth-types" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-chains", "alloy-consensus", @@ -10724,7 +10724,7 @@ dependencies = [ [[package]] name = "reth-rpc-layer" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-rpc-types-engine", "brotli", @@ -10746,7 +10746,7 @@ dependencies = [ [[package]] name = "reth-rpc-server-types" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -10774,7 +10774,7 @@ dependencies = [ [[package]] name = "reth-snap-sync" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -10815,7 +10815,7 @@ dependencies = [ [[package]] name = "reth-stages" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -10877,7 +10877,7 @@ dependencies = [ [[package]] name = "reth-stages-api" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10911,7 +10911,7 @@ dependencies = [ [[package]] name = "reth-stages-types" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-primitives", "arbitrary", @@ -10928,7 +10928,7 @@ dependencies = [ [[package]] name = "reth-static-file" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-primitives", "assert_matches", @@ -10951,7 +10951,7 @@ dependencies = [ [[package]] name = "reth-static-file-types" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-primitives", "clap", @@ -10969,7 +10969,7 @@ dependencies = [ [[package]] name = "reth-storage-api" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -10994,7 +10994,7 @@ dependencies = [ [[package]] name = "reth-storage-errors" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -11009,7 +11009,7 @@ dependencies = [ [[package]] name = "reth-storage-overlay" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -11039,7 +11039,7 @@ dependencies = [ [[package]] name = "reth-storage-rpc-provider" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -11069,7 +11069,7 @@ dependencies = [ [[package]] name = "reth-tasks" -version = "2.9.0" +version = "2.10.0" dependencies = [ "crossbeam-utils", "dashmap", @@ -11089,7 +11089,7 @@ dependencies = [ [[package]] name = "reth-testing-utils" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -11105,7 +11105,7 @@ dependencies = [ [[package]] name = "reth-tokio-util" -version = "2.9.0" +version = "2.10.0" dependencies = [ "tokio", "tokio-stream", @@ -11114,7 +11114,7 @@ dependencies = [ [[package]] name = "reth-tracing" -version = "2.9.0" +version = "2.10.0" dependencies = [ "clap", "eyre", @@ -11133,7 +11133,7 @@ dependencies = [ [[package]] name = "reth-tracing-otlp" -version = "2.9.0" +version = "2.10.0" dependencies = [ "base64 0.23.1", "clap", @@ -11141,8 +11141,11 @@ dependencies = [ "opentelemetry", "opentelemetry-appender-tracing", "opentelemetry-otlp", + "opentelemetry-proto", "opentelemetry-semantic-conventions", "opentelemetry_sdk", + "prost", + "reth-fs-util", "tracing", "tracing-opentelemetry", "tracing-subscriber", @@ -11151,7 +11154,7 @@ dependencies = [ [[package]] name = "reth-transaction-pool" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -11202,7 +11205,7 @@ dependencies = [ [[package]] name = "reth-trie" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -11212,6 +11215,7 @@ dependencies = [ "assert_matches", "auto_impl", "codspeed-criterion-compat", + "crossbeam-channel", "itertools 0.14.0", "metrics", "parking_lot", @@ -11224,6 +11228,7 @@ dependencies = [ "reth-primitives-traits", "reth-stages-types", "reth-storage-errors", + "reth-tasks", "reth-tracing", "reth-trie-common", "reth-trie-sparse", @@ -11233,7 +11238,7 @@ dependencies = [ [[package]] name = "reth-trie-common" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -11266,7 +11271,7 @@ dependencies = [ [[package]] name = "reth-trie-db" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -11291,44 +11296,9 @@ dependencies = [ "triehash", ] -[[package]] -name = "reth-trie-dst" -version = "0.1.0" -dependencies = [ - "alloy-consensus", - "alloy-primitives", - "alloy-rlp", - "alloy-trie", - "evm2", - "eyre", - "metrics-util", - "rand 0.9.5", - "reth-chain-state", - "reth-chainspec", - "reth-db", - "reth-db-api", - "reth-db-common", - "reth-engine-primitives", - "reth-engine-tree", - "reth-ethereum-primitives", - "reth-evm-ethereum", - "reth-execution-types", - "reth-node-builder", - "reth-node-core", - "reth-primitives-traits", - "reth-provider", - "reth-stages-types", - "reth-storage-overlay", - "reth-tasks", - "reth-trie", - "serde", - "serde_json", - "tempfile", -] - [[package]] name = "reth-trie-parallel" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-primitives", "crossbeam-channel", @@ -11352,7 +11322,7 @@ dependencies = [ [[package]] name = "reth-trie-sparse" -version = "2.9.0" +version = "2.10.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -11601,7 +11571,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -11681,7 +11651,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs 1.0.10", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -12642,7 +12612,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -13823,7 +13793,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index ad86bb8debd..d9ffb246277 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace.package] -version = "2.9.0" +version = "2.10.0" edition = "2024" rust-version = "1.96" license = "MIT OR Apache-2.0" @@ -11,7 +11,6 @@ exclude = [".github/"] members = [ "bin/reth-bb/", "bin/reth/", - "contrib/trie-dst/crates/reth-trie-dst/", "crates/storage/rpc-provider/", "crates/chain-state/", "crates/chainspec/", @@ -302,11 +301,6 @@ inherits = "release" debug = "full" strip = "none" -[profile.dst] -inherits = "profiling" -debug-assertions = true -overflow-checks = true - # Include debug info in benchmarks too. [profile.bench] inherits = "profiling" @@ -440,9 +434,9 @@ reth-trie-sparse = { path = "crates/trie/sparse", default-features = false } reth-zstd-compressors = { version = "0.8.2", default-features = false } # evm2 -evm2 = { git = "https://github.com/alloy-rs/evm2", rev = "f93318f120f6300599b21170d6c7bef81ed63d87", default-features = false } -evm2-inspectors = { git = "https://github.com/alloy-rs/evm2", rev = "f93318f120f6300599b21170d6c7bef81ed63d87", default-features = false } -evm2-jit = { git = "https://github.com/alloy-rs/evm2", rev = "f93318f120f6300599b21170d6c7bef81ed63d87", default-features = false } +evm2 = { git = "https://github.com/alloy-rs/evm2", rev = "592bf18c4e22fff54ac9b4db606c4ee276e345cb", default-features = false } +evm2-inspectors = { git = "https://github.com/alloy-rs/evm2", rev = "592bf18c4e22fff54ac9b4db606c4ee276e345cb", default-features = false } +evm2-jit = { git = "https://github.com/alloy-rs/evm2", rev = "592bf18c4e22fff54ac9b4db606c4ee276e345cb", default-features = false } # eth alloy-dyn-abi = "1.6.1" @@ -599,6 +593,8 @@ rocksdb = { version = "0.24", default-features = false, features = ["lz4", "zstd opentelemetry_sdk = "0.32" opentelemetry = "0.32" opentelemetry-otlp = "0.32" +opentelemetry-proto = { version = "0.32", default-features = false } +prost = { version = "0.14", default-features = false } opentelemetry-semantic-conventions = "0.32" opentelemetry-appender-tracing = "0.32" tracing-opentelemetry = "0.33" @@ -681,10 +677,13 @@ reth-firehose-tests = { path = "crates/firehose-tests" } # evm2 is patched through its pre-transfer URL, which redirects to alloy-rs/evm2, because Cargo # rejects a patch from the same source. -# [patch."https://github.com/alloy-rs/evm2"] -# evm2 = { git = "https://github.com/DaniPopes/evm2", rev = "f93318f120f6300599b21170d6c7bef81ed63d87" } -# evm2-inspectors = { git = "https://github.com/DaniPopes/evm2", rev = "f93318f120f6300599b21170d6c7bef81ed63d87" } -# evm2-jit = { git = "https://github.com/DaniPopes/evm2", rev = "f93318f120f6300599b21170d6c7bef81ed63d87" } +## Firehose additions +# SF fork of evm2 at the same upstream revision, which can run system calls (EIP-4788, EIP-2935, +# etc.) through the Inspector. +[patch."https://github.com/alloy-rs/evm2"] +evm2 = { git = "https://github.com/streamingfast/evm2", tag = "reth-v2.10.0-sf" } +evm2-inspectors = { git = "https://github.com/streamingfast/evm2", tag = "reth-v2.10.0-sf" } +evm2-jit = { git = "https://github.com/streamingfast/evm2", tag = "reth-v2.10.0-sf" } [patch.crates-io] # LLVM 23 support is not released yet. @@ -701,13 +700,6 @@ reth-primitives-traits = { git = "https://github.com/paradigmxyz/reth-core", rev reth-rpc-traits = { git = "https://github.com/paradigmxyz/reth-core", rev = "275ce4cf5befeb9f1e8acd935ff2882d2d25fc61" } reth-zstd-compressors = { git = "https://github.com/paradigmxyz/reth-core", rev = "275ce4cf5befeb9f1e8acd935ff2882d2d25fc61" } -## Firehose additions -# SF fork of evm2 that can run system calls (EIP-4788, EIP-2935, etc.) through the Inspector. -[patch."https://github.com/alloy-rs/evm2"] -evm2 = { git = "https://github.com/streamingfast/evm2", tag = "reth-v2.9.0-sf" } -evm2-inspectors = { git = "https://github.com/streamingfast/evm2", tag = "reth-v2.9.0-sf" } -evm2-jit = { git = "https://github.com/streamingfast/evm2", tag = "reth-v2.9.0-sf" } - ## Firehose additions # This is needed otherwise we get flooding of debug logs around a problem that seems to happen in Reth directly. # This is about wrong execution pool and could be related to ExEx in general (not just our version). We will diff --git a/bin/reth/tests/it/main.rs b/bin/reth/tests/it/main.rs index 68e0520d2c4..771013fefe0 100644 --- a/bin/reth/tests/it/main.rs +++ b/bin/reth/tests/it/main.rs @@ -1,5 +1,9 @@ #![allow(missing_docs)] +use alloy_node_bindings::Reth; +use alloy_primitives::{Address, U256}; +use alloy_provider::{Provider, ProviderBuilder}; +use alloy_rpc_types_eth::TransactionRequest; use std::{fs, path::Path, process::Command}; mod download; @@ -27,8 +31,6 @@ fn reth_ok(args: &[&str]) -> String { /// isolated. Each call gets a unique temporary data directory so that /// concurrent test runs never collide on the default `reth/dev/` path. fn spawn_dev() -> (alloy_node_bindings::RethInstance, tempfile::TempDir) { - use alloy_node_bindings::Reth; - let datadir = tempfile::tempdir().expect("failed to create temp dir"); let instance = Reth::at(RETH) @@ -97,8 +99,6 @@ fn unknown_flag() { #[tokio::test] async fn dev_node_eth_syncing() { - use alloy_provider::{Provider, ProviderBuilder}; - let (reth, _datadir) = spawn_dev(); let provider = ProviderBuilder::new().connect_http(reth.endpoint().parse().unwrap()); @@ -289,10 +289,6 @@ fn dump_genesis_sepolia_valid_json() { #[tokio::test] async fn dev_node_send_tx_and_mine() { - use alloy_primitives::{Address, U256}; - use alloy_provider::{Provider, ProviderBuilder}; - use alloy_rpc_types_eth::TransactionRequest; - let (reth, _datadir) = spawn_dev(); let provider = ProviderBuilder::new().connect_http(reth.endpoint().parse().unwrap()); diff --git a/contrib/trie-dst/.gitignore b/contrib/trie-dst/.gitignore deleted file mode 100644 index 2df455d9e9d..00000000000 --- a/contrib/trie-dst/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/crates/target/ diff --git a/contrib/trie-dst/README.md b/contrib/trie-dst/README.md deleted file mode 100644 index e15063e0e2f..00000000000 --- a/contrib/trie-dst/README.md +++ /dev/null @@ -1,258 +0,0 @@ -# Trie pipeline DST - -This workload drives reth's sparse-trie task and proof workers against a real -empty-state database and `OverlayManager`, with concurrent persistence through -`DatabaseProvider::save_blocks`. It does not start an execution client or execute -transactions. - -All harness files live here. The only production changes expose the existing -state-root task entrypoint, its options and pending-trie publication function, -the engine's changeset-cache retention constant, and the provider factory's -shared overlay manager. - -## Build and run locally - -From the reth repository root: - -```sh -cargo build --locked -p reth-trie-dst --profile dst -target/dst/reth-trie-dst '{"seed":42,"blocks":64,"datadir":"/tmp/trie-42"}' > /tmp/reth-trie-dst-42.log -``` - -Prerequisites for native builds are reth's normal Rust/native build dependencies, -including Clang for RocksDB's bindings. The crate belongs to the root workspace and uses its -shared dependencies, `Cargo.lock`, and the `dst` profile. Keep `dst` for this harness: -it inherits `profiling` while enabling debug assertions and overflow checks. - -The executable accepts one optional JSON object: - -| Field | Default | Range | -| --- | --- | --- | -| `seed` | 0 | unsigned 64-bit integer | -| `input` | generated from seed | explicit operation stream ending each block with `Commit` | -| `datadir` | temporary directory | persistent database and journal directory; reuse to restart | -| `blocks` | 64 | 1โ€“1024 | -| `updates` | 64 | 1โ€“4096 per block | -| `proof_workers` | 2 | 1โ€“16 per account/storage pool | -| `timeout_secs` | 60 | 1โ€“3600 per task result or persistence acknowledgement | -| `persistence_threshold` | 50 | retained canonical block count must exceed this to start persistence | -| `num_state_masking_blocks` | 30 | persisted suffix whose state/trie writes remain masked | -| `memory_block_buffer_target` | min(5, threshold) | blocks to retain above the database tip; optional/null uses Reth's default | - -These are the defaults in this Reth revision. As in Reth, the buffer must not -exceed the persistence threshold. With masking enabled, masking blocks plus the -buffer must be strictly less than the threshold. A zero threshold disables -masking. The backpressure threshold follows Reth's default, max(16, 2 ร— threshold). - -Each block's input operations and persistence frontiers are printed before its -result is awaited. The final JSON line reports completion, commits, partial -commits, and counts from production metrics. A mismatch or timeout exits -unsuccessfully. Keep stdout and stderr. - -## What runs - -The generator draws new addresses and storage slots uniformly from the full -32-bit range (0 through `0xFFFFFFFF`). It uses up to 16 generated accounts, -reusing existing accounts and slots for storage-only updates. It creates -nonempty accounts, changes nonce/balance, overwrites slots, and deletes -individual slots. Each block receives the configured number of -updates. It independently tracks the complete state and builds a per-block -`BundleState` with original and final account/storage values and reverts. - -The input is an ordered stream of `UpdateBatch`, `PrewarmBatch`, and `Commit` -operations. Each update batch contains account and storage updates in write -order; each prewarm batch contains account addresses and storage keys as -access hints. `Commit` finishes the sparse-trie task, checks the reference -root, and inserts the resulting block. The input records the batch order -explicitly, so replay needs no stream seed. Prewarm batches produce -`LeafUpdate::Touched` entries without changing state. -Hints include a random nonempty subset of distinct written keys plus read-only -accesses to unwritten accounts/storage. They overlap actual updates without -always covering them, and may arrive before or after an overlapping write. -Batch composition and hint targets are logged; the completion summary counts -update and hint batches. The generator uses the seed to construct the initial -stream. At each `Commit`, the root must equal a root rebuilt from the reference -maps with `HashBuilder`: -storage leaves first, followed by RLP account leaves. Its actual hashed state -and trie updates are attached to the synthetic executed block alongside the -bundle, then inserted into the real overlay manager. - -The task's pending trie is published only after the block root is accepted and -reused by subsequent tasks. Persistence acknowledgements request pruning by -the next task using the retained parent chain, newest first. - -Persistence follows the engine tree's linear-chain happy path. After a block -or persistence completion, an idle writer starts a save when the retained -canonical block count exceeds `persistence_threshold`. It saves block data -through `head - memory_block_buffer_target` and state/trie data through -`new_db_tip - num_state_masking_blocks`, never moving either frontier backwards. -The retained count includes the masked suffix, matching Reth's canonical -in-memory chain. Exactly one save is in flight; subsequent blocks continue -while it runs, with Reth's normal backpressure limit. Completion is polled at -block boundaries. Finish advances through the production write path. Only after -acknowledgement are persisted overlays removed through the state/trie frontier, -the execution overlay precomputed at its new anchor, and trie pruning requested. -At shutdown, the worker finishes any active save and fully persists the head, -matching Reth's shutdown flush. -It then drains trie, proof-worker, and deferred-drop queues before exiting, so -all database handles close while RocksDB's global scheduler is still alive. - -After every persistence commit, a fresh overlay-backed provider must expose the -reference accounts and every touched slot, including deleted slots, plus both -range endpoints at the database tip. Fully flushed commits additionally compare -the complete raw hashed tables with the reference and calculate a fresh -database-backed root. Metrics report proof revelation, trie reuse, pruning, -overlay computation, and partial commits. Their counts depend on the selected -persistence settings and run length. - -The chain is linear. Reorgs, EVM execution, arbitrary account -destruction and storage wipes are excluded. In particular, the generator does -not invent storage wipes outside valid CREATE2/SELFDESTRUCT semantics. - -## Run under Bedrock - -Use a Bedrock checkout with the generic workload runner and its documented host -requirements (Docker, Nix, and the Bedrock kernel/device setup). From that checkout: - -```sh -RUN_DIR=/tmp/reth-trie-inputs ./scripts/dst/run.sh \ - --workload /absolute/path/to/reth/contrib/trie-dst/workload \ - --workload-arg blocks=64 --workload-arg proof_workers=2 \ - --duration 10m --workers 16 --out /tmp/reth-trie-runs -``` - -The preparation hook builds the DST executable and controller image, -compiles the readiness helper against Bedrock's `guest/libvmcall.h`, and -normalizes the image archive with Bedrock's archive tool. It keeps symbols in -the original executable and strips debug sections only from the image copy. -The inline builder supplies Rust, Clang, sccache, and mold; the host needs Docker, -Python 3, and `flock` on x86-64 Linux. `DOCKER` and `CARGO_BUILD_JOBS` customize -the build tools. - -Cargo downloads, compiled targets, and the unstripped executable persist under -`BEDROCK_DST_BUILD_CACHE` (Bedrock CI's workload cache), defaulting to -`${XDG_CACHE_HOME:-$HOME/.cache}/reth-trie-dst`. Compilation uses sccache and mold -with the `dst` profile. `BEDROCK_DST_SCCACHE_DIR` selects the compiler cache, defaulting -to the workload cache's `sccache` directory; `SCCACHE_CACHE_SIZE` defaults to `10G`. -Preparation writes timings and cache paths to `RUN_DIR/cache-info.env` and -compiler statistics to `RUN_DIR/sccache.txt`, which Bedrock CI displays and -uploads with its artifacts. - -The controller implements `parse-args`, `config`, `warmup`, `start`, `progress`, -`finalize` and `observe`. It starts the executable under Bedrock's `thread-fuzz`, -so the scheduler varies the real task/proof/persistence interleavings. By default, -each seed resumes the same live initialized worker and empty database from the -shared warm checkpoint; `cold_start: true` instead creates them after branching. The seed -controls generated operations and the recorded crash plan; Bedrock's recorded -random/preemption streams control scheduling. Mid-run semantic decision editing -is unsupported. - -The Bedrock-only `crashes` argument defaults to 1 (range 0โ€“16, at most -`blocks`). Set it to 0 for the happy path. The controller draws distinct block -numbers uniformly across the entire run, from block 1 through the final block. -When it observes the corresponding block-input event, it sends SIGKILL to the Rust -process and immediately restarts it against the same directory. The worker -continues running after emitting that event, so the kill can interrupt trie -computation or concurrent persistence. These are process crashes: the kernel -and its page cache survive. Container restarts and power-loss faults are outside -this workload. - -Before processing each new block, Rust appends its operations to `blocks.jsonl`. -Startup reads the journal into block-number order, deduplicates identical records, -rejects conflicting records, and ignores incomplete JSON left by an interrupted -append. A new append starts on a fresh line. Operations are read back rather than -regenerated, including blocks that must be replayed after an unwind. - -Startup uses `LaunchContext::create_provider_factory_with_recovery`, the real node -startup path. It checks storage consistency and unwinds the production pipeline -when `partial_state_trie < Finish`. Reth's durable unwind marker also handles a -second crash during recovery. The actual committed checkpoint determines the -recovered head, including commits whose acknowledgement was lost. The reference -state is reconstructed from the journal through that head and compared with the -recovered header root, hashed tables, and storage. All -volatile overlays and the sparse trie start cold; generation then resumes at the -next block. Keep the same seed and configuration when reopening a directory. - -The run log records pre-recovery Finish/state-trie frontiers, the recovered head -and root, each kill, and the existing block/persistence events. `done.json` reports -armed and fired kill counts and completed recoveries. A missed kill, unexpected -process exit, failed recovery check, or failure to finish within the seed horizon -fails the run. - -Finalization reports unfinished work as a failure and collects `reth-trie-dst-run.log`, -`reth-trie-dst-controller.log`, `reth-trie-dst-done.json`, and the append-only -`reth-trie-dst-blocks.jsonl` under `/bedrock/out` alongside -the merged assertion stream. A worker timeout also fails the run. Increase the -driver's per-seed virtual horizon (`--run-secs`) if a larger configuration cannot -finish before finalization. - -Bedrock's shared runner owns the host wall-clock budget: `--duration 10m` -includes guest boot and warmup, and optional `--num-seeds 100` caps the total -seeds across all workers. Omit the cap to run until the deadline. Each seed still -executes its configured number of blocks. The deadline stops in-flight seeds and -reports them as interrupted; they do not count as passing. - -For CI, use Bedrock's DST workflow with this external workload source, replacing -`RETH_REF` with a branch, tag, or commit: - -```sh -gh workflow run tempo-e2e.yml --repo tempoxyz/bedrock \ - -f workload='git::https://github.com/paradigmxyz/reth.git//contrib/trie-dst/workload?ref=RETH_REF' \ - -f workload_config='{}' -f duration=10m -f parallel_seeds=16 -``` - -CI's `duration` is the same host budget; `num_seeds` is a global cap, unlimited -when blank. The workflow fetches the selected Reth revision. Preserve `RUN_DIR` -and use the generic runner's `--prepared` option to reuse the exact inputs. For an actual recorded schedule, use Bedrock's replay command on the -recorded run; rebuilding an image or changing the binary changes the World. - -## Fuzz explicit trie inputs - -With the accompanying Bedrock workload-input hooks, set `DST_CMD=fuzz` during -preparation and execution. For example, dispatch the same CI workflow with -`mode=fuzz`, `parallel_seeds=1`, and this workload URL. The workflow's -`num_seeds` then caps fuzz executions. Each corpus entry is an `Input`: an -ordered operation stream. Reth generates eight initial entries and exposes -typed AST sites for LibAFL to choose. Its operators edit account/storage -fields, prewarm keys, nested updates, batches, and commit boundaries; crossover -draws a second corpus entry. Invalid edits leave the input unchanged. The same input can -also be supplied as `base_input` to the controller for replay. - -At each completed block, the controller emits `trie/input/checked`, which gives -Bedrock a pull point when mid-run snapshots are enabled. A suffix input is -injected at the next new block boundary; already journaled blocks are never -replaced. The worker waits after its original input until Bedrock either injects -a suffix or finalizes the run. This workload sets `--max-snaps=0` in -`bedrock.yaml`, so fuzzing currently uses only whole-input mutations from the -warm checkpoint; the suffix path remains available for experiments. Fuzz -preparation builds a coverage-instrumented guest executable and keeps an -uninstrumented executable beside the host controller for input hooks. -The first scope is trie operation streams on a linear -chain; it does not mutate Reth code or the VM's recorded scheduling decisions. - -### Warm initialization - -The default controller warmup starts a live worker from fixed `workload_args`, -initializes its runtime, empty database, storage and genesis, and checks a -Unix-datagram readiness handshake before returning `ready: true`. Repeated hooks -poll the same process; dead workers and initialization exceeding 120 guest seconds -are errors. The worker blocks in bounded socket I/O (one-hour guest deadline), not -an input-dependent polling loop. No corpus input, generation or expected roots are -loaded in this warm prefix. - -After branching, `start` sends the execution config path over that socket. The -worker validates that proof-worker counts, storage directory and persistence/tree -settings match the initialized configuration, then parses and runs the input and -its normal initial assertions. Guest-local `stdout.log` retains output across hook -processes; `init.log` and `warm-ready.json` capture the idle worker PID and output -frontier. Adoption seeks past that frontier and emits a `warm: reused` PID event, -so pre-checkpoint recovery output is not counted as per-execution recovery; `run.log` and the controller's SIGKILL/recovery monitor remain active. -Subsequent crash restarts launch normally and reopen the real persisted database, -without an initialization barrier. Continuation input and synchronous -`run_secs: 1` execution use the same monitor as longer runs. - -The current coverage frontend retains inline counters and its crash-persistent -feedback map across the warm checkpoint; the initialization barrier alone does -not remove initialization coverage from per-input feedback. - -Set `cold_start: true` in workload args to retain per-execution startup coverage. diff --git a/contrib/trie-dst/crates/reth-trie-dst/Cargo.toml b/contrib/trie-dst/crates/reth-trie-dst/Cargo.toml deleted file mode 100644 index 8338e5faeb8..00000000000 --- a/contrib/trie-dst/crates/reth-trie-dst/Cargo.toml +++ /dev/null @@ -1,43 +0,0 @@ -[package] -name = "reth-trie-dst" -version = "0.1.0" -edition.workspace = true -license.workspace = true -publish = false - -[lints] -workspace = true - -[features] -inline-coverage = [] - -[dependencies] -reth-chain-state.workspace = true -reth-chainspec.workspace = true -reth-db-api.workspace = true -reth-db-common.workspace = true -reth-db.workspace = true -reth-engine-primitives.workspace = true -reth-engine-tree.workspace = true -reth-ethereum-primitives.workspace = true -reth-evm-ethereum.workspace = true -reth-execution-types.workspace = true -reth-node-builder.workspace = true -reth-node-core.workspace = true -reth-primitives-traits.workspace = true -reth-provider = { workspace = true, features = ["test-utils"] } -reth-stages-types.workspace = true -reth-storage-overlay = { workspace = true, features = ["rayon"] } -reth-tasks = { workspace = true, features = ["rayon", "test-utils"] } -reth-trie.workspace = true -alloy-consensus.workspace = true -alloy-primitives = { workspace = true, features = ["asm-keccak"] } -alloy-rlp.workspace = true -alloy-trie.workspace = true -evm2 = { workspace = true, features = ["std"] } -eyre.workspace = true -metrics-util = { workspace = true, features = ["debugging"] } -rand.workspace = true -serde = { workspace = true, features = ["derive"] } -serde_json.workspace = true -tempfile.workspace = true diff --git a/contrib/trie-dst/crates/reth-trie-dst/src/input.rs b/contrib/trie-dst/crates/reth-trie-dst/src/input.rs deleted file mode 100644 index 1d157e00601..00000000000 --- a/contrib/trie-dst/crates/reth-trie-dst/src/input.rs +++ /dev/null @@ -1,492 +0,0 @@ -//! Explicit trie-task operation stream. Only `Commit` ends a block. - -use crate::model::{apply, generate, random_address, random_slot, Model, Update}; -use alloy_primitives::{Address, B256, KECCAK256_EMPTY, U256}; -use eyre::{ensure, Result}; -use rand::{rngs::StdRng, seq::SliceRandom, Rng, SeedableRng}; -use serde::{Deserialize, Serialize}; -use std::collections::BTreeSet; - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct Input { - pub operations: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct StorageHint { - pub address: Address, - pub slot: U256, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub(crate) enum Operation { - UpdateBatch { updates: Vec }, - PrewarmBatch { accounts: Vec
, storages: Vec }, - Commit, -} - -#[derive(Serialize)] -pub(crate) struct MutationSite { - pub operator: &'static str, - pub path: Vec, -} - -impl Input { - pub(crate) fn generate(seed: u64, blocks: u64, updates: usize) -> Self { - let mut state = Model::default(); - let mut operations = Vec::new(); - for number in 1..=blocks { - let mut rng = StdRng::seed_from_u64(seed.wrapping_add(number)); - let generated = generate(&mut rng, &state, updates); - apply(&mut state, number, &generated); - let mut targets = generated - .iter() - .map(|update| match update { - Update::Account { address, .. } => (*address, None), - Update::Storage { address, slot, .. } => (*address, Some(*slot)), - }) - .collect::>(); - targets.shuffle(&mut rng); - let count = - if targets.len() > 1 { rng.random_range(1..targets.len()) } else { targets.len() }; - targets.truncate(count); - for _ in 0..rng.random_range(1..=8) { - let address = random_address(&mut rng); - let slot = rng.random_bool(0.5).then(|| random_slot(&mut rng)); - targets.push((address, slot)); - } - targets.shuffle(&mut rng); - let mut targets = targets.into_iter().peekable(); - let mut updates = generated.into_iter().peekable(); - while targets.peek().is_some() || updates.peek().is_some() { - if targets.peek().is_some() && (updates.peek().is_none() || rng.random_bool(0.5)) { - let size = rng.random_range(1..=targets.len().min(16)); - let mut accounts = Vec::new(); - let mut storages = Vec::new(); - for (address, slot) in targets.by_ref().take(size) { - match slot { - Some(slot) => storages.push(StorageHint { address, slot }), - None => accounts.push(address), - } - } - operations.push(Operation::PrewarmBatch { accounts, storages }); - } else { - let size = rng.random_range(1..=updates.len().min(16)); - operations.push(Operation::UpdateBatch { - updates: updates.by_ref().take(size).collect(), - }); - } - } - operations.push(Operation::Commit); - } - Self { operations } - } - - pub(crate) fn blocks(&self) -> Vec> { - self.operations - .split_inclusive(|op| matches!(op, Operation::Commit)) - .map(<[Operation]>::to_vec) - .collect() - } - - pub(crate) fn validate(&self) -> Result<()> { - let mut accounts = BTreeSet::new(); - let mut blocks = 0; - let mut block_updates = 0; - for operation in &self.operations { - match operation { - Operation::UpdateBatch { updates } => { - ensure!(!updates.is_empty(), "empty update batch"); - block_updates += updates.len(); - ensure!(block_updates <= 4096, "block needs at most 4096 updates"); - for update in updates { - match update { - Update::Account { address, account } => { - ensure!(!account.is_empty(), "empty account: {address}"); - ensure!( - account.bytecode_hash != Some(KECCAK256_EMPTY), - "noncanonical empty code hash: {address}" - ); - accounts.insert(*address); - } - Update::Storage { address, .. } => { - ensure!( - accounts.contains(address), - "storage before account creation: {address}" - ); - } - } - } - } - Operation::PrewarmBatch { accounts, storages } => { - ensure!(!accounts.is_empty() || !storages.is_empty(), "empty prewarm batch"); - } - Operation::Commit => { - ensure!(block_updates > 0, "commit without updates"); - blocks += 1; - ensure!(blocks <= 1024, "input needs at most 1024 commits"); - block_updates = 0; - } - } - } - ensure!( - matches!(self.operations.last(), Some(Operation::Commit)), - "input must end with a commit" - ); - Ok(()) - } - - pub(crate) fn mutation_sites(&self, only: Option<&str>) -> Vec { - let mut sites = Vec::new(); - for (i, op) in self.operations.iter().enumerate() { - let root = vec!["operations".into(), i.to_string()]; - match op { - Operation::UpdateBatch { updates } => { - if only.is_none_or(|name| { - name.starts_with("update.") || - name.starts_with("account.") || - name.starts_with("storage.") - }) { - for (j, update) in updates.iter().enumerate() { - let mut path = root.clone(); - path.extend(["updates".into(), j.to_string()]); - for operator in - ["update.duplicate", "update.remove", "update.crossover"] - { - if only.is_none_or(|name| name == operator) { - sites.push(MutationSite { operator, path: path.clone() }); - } - } - let (variant, fields): (&str, &[&str]) = match update { - Update::Account { .. } => ( - "Account", - &[ - "address", - "account/balance", - "account/nonce", - "account/bytecode_hash", - ], - ), - Update::Storage { .. } => { - ("Storage", &["address", "slot", "value"]) - } - }; - for field in fields { - let operator = match (variant, *field) { - ("Account", "address") => "account.address", - ("Account", "account/balance") => "account.balance", - ("Account", "account/nonce") => "account.nonce", - ("Account", _) => "account.bytecode_hash", - ("Storage", "address") => "storage.address", - ("Storage", "slot") => "storage.slot", - _ => "storage.value", - }; - if only.is_none_or(|name| name == operator) { - let mut field_path = path.clone(); - field_path.push(variant.into()); - field_path.extend(field.split('/').map(str::to_string)); - sites.push(MutationSite { operator, path: field_path }); - } - } - } - } - if only.is_none_or(|name| name == "commit.insert") { - sites.push(MutationSite { operator: "commit.insert", path: root.clone() }); - } - } - Operation::PrewarmBatch { accounts, storages } => { - if only.is_none_or(|name| name == "prewarm.account") { - for j in 0..accounts.len() { - let mut path = root.clone(); - path.extend(["accounts".into(), j.to_string()]); - sites.push(MutationSite { operator: "prewarm.account", path }); - } - } - if only.is_none_or(|name| name.starts_with("prewarm.storage.")) { - for j in 0..storages.len() { - let mut path = root.clone(); - path.extend(["storages".into(), j.to_string()]); - for (operator, field) in [ - ("prewarm.storage.address", "address"), - ("prewarm.storage.slot", "slot"), - ] { - if only.is_none_or(|name| name == operator) { - let mut field_path = path.clone(); - field_path.push(field.into()); - sites.push(MutationSite { operator, path: field_path }); - } - } - } - } - } - Operation::Commit => { - if i + 1 < self.operations.len() && - only.is_none_or(|name| name == "commit.remove") - { - sites.push(MutationSite { operator: "commit.remove", path: root }); - } - continue; - } - } - for operator in ["batch.duplicate", "batch.remove", "batch.crossover"] { - if only.is_none_or(|name| name == operator) { - sites.push(MutationSite { operator, path: root.clone() }); - } - } - } - sites - } - - pub(crate) fn mutate_at( - &mut self, - operator: &str, - path: &[String], - seed: u64, - donor: Option<(&Self, &[String])>, - ) -> Result<()> { - self.validate()?; - let original = self.clone(); - let mut rng = StdRng::seed_from_u64(seed); - ensure!(path.first().map(String::as_str) == Some("operations"), "invalid mutation path"); - let i: usize = - path.get(1).ok_or_else(|| eyre::eyre!("missing operation index"))?.parse()?; - match operator { - "commit.insert" => { - ensure!( - matches!(self.operations.get(i), Some(Operation::UpdateBatch { .. })), - "commit insertion needs an update batch" - ); - self.operations.insert(i + 1, Operation::Commit); - } - "commit.remove" => { - ensure!( - i + 1 < self.operations.len() && - matches!(self.operations.get(i), Some(Operation::Commit)), - "cannot remove final commit" - ); - self.operations.remove(i); - } - "batch.duplicate" => { - let op = self - .operations - .get(i) - .ok_or_else(|| eyre::eyre!("operation out of range"))? - .clone(); - ensure!(!matches!(op, Operation::Commit), "cannot duplicate commit"); - self.operations.insert(i, op); - } - "batch.remove" => { - ensure!( - !matches!(self.operations.get(i), Some(Operation::Commit) | None), - "cannot remove commit" - ); - self.operations.remove(i); - } - "batch.crossover" => { - let (other, other_path) = donor.ok_or_else(|| eyre::eyre!("missing donor"))?; - ensure!( - other_path.first().map(String::as_str) == Some("operations"), - "invalid donor path" - ); - let j: usize = - other_path.get(1).ok_or_else(|| eyre::eyre!("missing donor index"))?.parse()?; - let replacement = other - .operations - .get(j) - .ok_or_else(|| eyre::eyre!("donor out of range"))? - .clone(); - ensure!(!matches!(replacement, Operation::Commit), "cannot cross commit"); - *self - .operations - .get_mut(i) - .ok_or_else(|| eyre::eyre!("operation out of range"))? = replacement; - } - "update.duplicate" | "update.remove" | "update.crossover" => { - ensure!(path.get(2).map(String::as_str) == Some("updates"), "invalid update path"); - let j: usize = - path.get(3).ok_or_else(|| eyre::eyre!("missing update index"))?.parse()?; - let replacement = if operator == "update.crossover" { - let (other, other_path) = donor.ok_or_else(|| eyre::eyre!("missing donor"))?; - ensure!( - other_path.first().map(String::as_str) == Some("operations") && - other_path.get(2).map(String::as_str) == Some("updates"), - "invalid donor update path" - ); - let oi: usize = other_path - .get(1) - .ok_or_else(|| eyre::eyre!("missing donor operation"))? - .parse()?; - let oj: usize = other_path - .get(3) - .ok_or_else(|| eyre::eyre!("missing donor update"))? - .parse()?; - let Some(Operation::UpdateBatch { updates }) = other.operations.get(oi) else { - eyre::bail!("donor is not an update batch") - }; - Some( - updates - .get(oj) - .ok_or_else(|| eyre::eyre!("donor update out of range"))? - .clone(), - ) - } else { - None - }; - let Some(Operation::UpdateBatch { updates }) = self.operations.get_mut(i) else { - eyre::bail!("not an update batch") - }; - ensure!(j < updates.len(), "update out of range"); - match operator { - "update.duplicate" => updates.insert(j, updates[j].clone()), - "update.remove" => { - updates.remove(j); - } - _ => updates[j] = replacement.expect("crossover donor"), - } - } - "account.address" | - "account.balance" | - "account.nonce" | - "account.bytecode_hash" | - "storage.address" | - "storage.slot" | - "storage.value" => { - ensure!(path.get(2).map(String::as_str) == Some("updates"), "invalid update path"); - let j: usize = - path.get(3).ok_or_else(|| eyre::eyre!("missing update index"))?.parse()?; - let available_accounts = if operator == "storage.address" { - self.operations - .iter() - .take(i + 1) - .enumerate() - .filter_map(|(index, op)| match op { - Operation::UpdateBatch { updates } => { - Some(updates.iter().take(if index == i { - j - } else { - updates.len() - })) - } - _ => None, - }) - .flatten() - .filter_map(|update| match update { - Update::Account { address, .. } => Some(*address), - _ => None, - }) - .collect::>() - } else { - Vec::new() - }; - let mut renamed = None; - let Some(Operation::UpdateBatch { updates }) = self.operations.get_mut(i) else { - eyre::bail!("not an update batch") - }; - match updates.get_mut(j) { - Some(Update::Account { address, account }) - if operator.starts_with("account.") => - { - match operator { - "account.address" => { - let old = *address; - *address = random_address(&mut rng); - renamed = Some((old, *address)); - } - "account.balance" => account.balance = U256::from(rng.random::()), - "account.nonce" => account.nonce = rng.random(), - _ => account.bytecode_hash = Some(B256::from(rng.random::<[u8; 32]>())), - } - } - Some(Update::Storage { address, slot, value }) - if operator.starts_with("storage.") => - { - match operator { - "storage.address" => { - if !available_accounts.is_empty() { - *address = available_accounts - [rng.random_range(0..available_accounts.len())]; - } - } - "storage.slot" => *slot = random_slot(&mut rng), - _ => *value = U256::from(rng.random::()), - } - } - _ => eyre::bail!("update path/type mismatch"), - } - if let Some((old, new)) = renamed { - // Storage after this account update belongs to its new address until - // recreation. - 'operations: for (index, op) in self.operations.iter_mut().enumerate().skip(i) { - if let Operation::UpdateBatch { updates } = op { - for (update_index, update) in updates.iter_mut().enumerate() { - if index == i && update_index <= j { - continue; - } - match update { - Update::Account { address, .. } if *address == old => { - break 'operations; - } - Update::Storage { address, .. } if *address == old => { - *address = new; - } - _ => {} - } - } - } - } - } - } - "prewarm.account" | "prewarm.storage.address" | "prewarm.storage.slot" => { - let j: usize = - path.get(3).ok_or_else(|| eyre::eyre!("missing hint index"))?.parse()?; - let Some(Operation::PrewarmBatch { accounts, storages }) = - self.operations.get_mut(i) - else { - eyre::bail!("not a prewarm batch") - }; - if operator == "prewarm.account" { - ensure!( - path.get(2).map(String::as_str) == Some("accounts"), - "invalid account hint path" - ); - *accounts - .get_mut(j) - .ok_or_else(|| eyre::eyre!("account hint out of range"))? = - random_address(&mut rng); - } else { - ensure!( - path.get(2).map(String::as_str) == Some("storages"), - "invalid storage hint path" - ); - let hint = storages - .get_mut(j) - .ok_or_else(|| eyre::eyre!("storage hint out of range"))?; - if operator == "prewarm.storage.address" { - hint.address = random_address(&mut rng); - } else { - hint.slot = random_slot(&mut rng); - } - } - } - _ => eyre::bail!("unknown mutation operator: {operator}"), - } - if self.validate().is_err() { - *self = original; - } - Ok(()) - } - - pub(crate) fn mutate(&mut self, seed: u64) -> Result<()> { - let sites = self - .mutation_sites(None) - .into_iter() - .filter(|site| !site.operator.ends_with(".crossover")) - .collect::>(); - let mut rng = StdRng::seed_from_u64(seed); - let site = &sites[rng.random_range(0..sites.len())]; - self.mutate_at(site.operator, &site.path, rng.random(), None) - } -} diff --git a/contrib/trie-dst/crates/reth-trie-dst/src/main.rs b/contrib/trie-dst/crates/reth-trie-dst/src/main.rs deleted file mode 100644 index 6e4372ea633..00000000000 --- a/contrib/trie-dst/crates/reth-trie-dst/src/main.rs +++ /dev/null @@ -1,1009 +0,0 @@ -//! Bounded workload for the sparse-trie, proof, overlay and persistence pipeline. - -mod input; -mod model; - -use alloy_consensus::Header; -use alloy_primitives::{keccak256, B256, U256}; -use eyre::{ensure, Context, Result}; -use input::{Input, Operation}; -use metrics_util::debugging::{DebugValue, DebuggingRecorder, Snapshotter}; -use model::{apply, root, Model}; -use reth_chain_state::ExecutedBlock; -use reth_chainspec::ChainSpec; -use reth_db::{init_db, DatabaseEnv}; -use reth_db_api::{cursor::DbCursorRO, tables, transaction::DbTx}; -use reth_engine_primitives::MIN_PERSISTENCE_BACKPRESSURE_THRESHOLD; -use reth_engine_tree::tree::{ - state_root_strategy::{ - publish_sparse_trie, DefaultStateRootStrategy, StateAccessHint, StateRootTaskOptions, - }, - TreeConfig, CHANGESET_CACHE_RETENTION_BLOCKS, -}; -use reth_ethereum_primitives::{Block, EthPrimitives}; -use reth_evm_ethereum::EthEvmConfig; -use reth_execution_types::{BlockExecutionOutput, BlockExecutionResult}; -use reth_node_builder::common::{LaunchContext, WithConfigs}; -use reth_node_core::{ - dirs::{ChainPath, DataDirPath}, - node_config::NodeConfig, -}; -use reth_primitives_traits::{AlloyBlockHeader, RecoveredBlock, SealedHeader}; -use reth_provider::{ - providers::LatestStateProviderRef, test_utils::MockNodeTypesWithDB, AccountReader, - DatabaseProviderROFactory, HeaderProvider, ProviderFactory, SaveBlocksInput, - StageCheckpointReader, StateProvider, StateRootProvider, -}; -use reth_stages_types::StageId; -use reth_storage_overlay::{OverlayManager, OverlayStateProviderFactory}; -use reth_tasks::{RayonConfig, RuntimeBuilder, RuntimeConfig, TokioConfig}; -use reth_trie::{HashedPostState, SortedTrieData}; -use serde::{Deserialize, Serialize}; -use serde_json::json; -use std::{ - collections::{btree_map::Entry, BTreeMap, BTreeSet}, - fs::{self, OpenOptions}, - io::{Read, Write}, - os::unix::{fs::MetadataExt, net::UnixDatagram, process::CommandExt}, - path::PathBuf, - sync::{mpsc, Arc}, - time::{Duration, Instant}, -}; - -type Node = MockNodeTypesWithDB>; - -#[cfg(feature = "inline-coverage")] -unsafe extern "C" { - fn bedrock_coverage_flush(); -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -struct JournalBlock { - number: u64, - operations: Vec, -} - -#[derive(Clone, Debug, Deserialize)] -#[serde(default, deny_unknown_fields)] -struct Config { - seed: u64, - input: Option, - expected_roots: Option>, - /// An atomic mailbox for snapshot continuations, applied before the next new block. - input_file: Option, - datadir: Option, - /// Guest-local initialization barrier, used only by the first warm process. - init_socket: Option, - blocks: u64, - updates: usize, - proof_workers: usize, - timeout_secs: u64, - persistence_threshold: u64, - num_state_masking_blocks: u64, - memory_block_buffer_target: Option, -} - -impl Default for Config { - fn default() -> Self { - let tree = TreeConfig::default(); - Self { - seed: 0, - input: None, - expected_roots: None, - input_file: None, - datadir: None, - init_socket: None, - blocks: 64, - updates: 64, - proof_workers: 2, - timeout_secs: 60, - persistence_threshold: tree.persistence_threshold(), - num_state_masking_blocks: tree.num_state_masking_blocks(), - memory_block_buffer_target: None, - } - } -} - -#[allow(clippy::clone_on_copy, reason = "Account is not Copy with account-ext enabled.")] -fn verify(factory: &ProviderFactory, state: &Model, tip: u64, state_tip: u64) -> Result<()> { - let provider = factory.provider()?; - let checkpoint = provider - .get_stage_checkpoint(StageId::Finish)? - .ok_or_else(|| eyre::eyre!("missing Finish checkpoint"))?; - ensure!(checkpoint.block_number == tip, "Finish != {tip}"); - let actual_state_tip = checkpoint - .finish_stage_checkpoint() - .and_then(|finish| finish.partial_state_trie()) - .unwrap_or(checkpoint.block_number); - ensure!(actual_state_tip == state_tip, "state/trie frontier != {state_tip}"); - let hash = - provider.sealed_header(tip)?.ok_or_else(|| eyre::eyre!("missing header {tip}"))?.hash(); - drop(provider); - let view = OverlayStateProviderFactory::new( - factory.clone(), - factory.overlay_manager().overlay_builder(hash), - ) - .database_provider_ro()?; - for (address, entry) in state { - ensure!( - view.basic_account(address)?.as_ref() == Some(&entry.account), - "plain account {address} at {tip}" - ); - // Include deleted slots and both range endpoints, not only surviving leaves. - for slot in entry - .touched_slots - .iter() - .copied() - .chain([U256::ZERO, U256::from(u32::MAX)]) - .map(B256::from) - .collect::>() - { - ensure!( - view.storage(*address, slot)?.unwrap_or_default() == - entry.storage.get(&U256::from_be_bytes(slot.0)).copied().unwrap_or_default(), - "plain storage {address}/{slot} at {tip}" - ); - } - } - drop(view); - if tip == state_tip { - let expected_root = root(state); - let provider = factory.provider()?; - let accounts = provider - .tx_ref() - .cursor_read::()? - .walk(None)? - .collect::, _>>()?; - let expected = state - .iter() - .map(|(address, entry)| (keccak256(address), entry.account.clone())) - .collect::>(); - ensure!(accounts == expected, "persisted hashed accounts at {tip}"); - let storage = provider - .tx_ref() - .cursor_read::()? - .walk(None)? - .map(|row| row.map(|(address, entry)| ((address, entry.key), entry.value))) - .collect::, _>>()?; - let expected = state - .iter() - .flat_map(|(address, entry)| { - entry.storage.iter().map(move |(slot, value)| { - ((keccak256(address), keccak256(B256::from(*slot))), *value) - }) - }) - .collect::>(); - ensure!(storage == expected, "persisted hashed storage at {tip}"); - let latest = LatestStateProviderRef::new(&provider); - ensure!( - latest.state_root(HashedPostState::default())? == expected_root, - "persisted root at {tip}" - ); - } - Ok(()) -} - -fn metric_counts(snapshotter: &Snapshotter) -> BTreeMap { - snapshotter - .snapshot() - .into_vec() - .into_iter() - .filter_map(|(key, _, _, value)| { - let name = key.key().name().to_string(); - if !(name.contains("tree.root") || - name.contains("state_trie_overlay") || - name.contains("storage.overlay")) - { - return None; - } - let count = match value { - DebugValue::Counter(count) => count, - DebugValue::Histogram(values) => values.len() as u64, - _ => return None, - }; - Some((name, count)) - }) - .collect() -} - -fn run(config: &mut Config, snapshotter: &Snapshotter) -> Result<()> { - ensure!((1..=1024).contains(&config.blocks), "blocks must be 1..=1024"); - ensure!((1..=4096).contains(&config.updates), "updates must be 1..=4096"); - ensure!((1..=16).contains(&config.proof_workers), "proof_workers must be 1..=16"); - ensure!((1..=3600).contains(&config.timeout_secs), "timeout_secs must be 1..=3600"); - let buffer = config.memory_block_buffer_target.unwrap_or_else(|| { - TreeConfig::default().memory_block_buffer_target().min(config.persistence_threshold) - }); - let masking = - if config.persistence_threshold == 0 { 0 } else { config.num_state_masking_blocks }; - let backpressure = - config.persistence_threshold.saturating_mul(2).max(MIN_PERSISTENCE_BACKPRESSURE_THRESHOLD); - ensure!(backpressure > config.persistence_threshold, "persistence threshold is too large"); - ensure!(buffer <= config.persistence_threshold, "buffer must not exceed persistence threshold"); - ensure!( - masking == 0 || - matches!(masking.checked_add(buffer), Some(window) if window < config.persistence_threshold), - "masking blocks + buffer must be less than persistence threshold" - ); - // Clear the default window before applying smaller thresholds: TreeConfig checks each setter. - let tree_config = TreeConfig::default() - .with_num_state_masking_blocks(0) - .with_memory_block_buffer_target(0) - .with_persistence_backpressure_threshold(u64::MAX) - .with_persistence_threshold(config.persistence_threshold) - .with_num_state_masking_blocks(masking) - .with_memory_block_buffer_target(buffer) - .with_persistence_backpressure_threshold(backpressure); - let runtime = RuntimeBuilder::new(RuntimeConfig { - tokio: TokioConfig::with_worker_threads(2), - rayon: RayonConfig { - cpu_threads: Some(2), - rpc_threads: Some(1), - storage_threads: Some(2), - proof_storage_worker_threads: Some(config.proof_workers), - proof_account_worker_threads: Some(config.proof_workers), - prewarming_threads: Some(1), - bal_streaming_threads: Some(1), - state_trie_overlay_worker_threads: Some(1), - ..Default::default() - }, - }) - .build()?; - // A supplied directory survives process death; standalone runs default to a temporary one. - let temporary = config.datadir.is_none().then(tempfile::tempdir).transpose()?; - let datadir = - config.datadir.clone().unwrap_or_else(|| temporary.as_ref().unwrap().path().to_path_buf()); - fs::create_dir_all(&datadir)?; - let chain = Arc::new(ChainSpec::default()); - let data_dir = ChainPath::::new( - datadir.to_string_lossy().parse()?, - chain.chain, - Default::default(), - ); - let db = Arc::new(init_db(data_dir.db(), Default::default())?); - let context = LaunchContext::new(runtime.clone(), data_dir) - .with(WithConfigs { config: NodeConfig::new(chain), toml_config: Default::default() }) - .attach(db); - println!("{}", json!({"recovery": "start"})); - // This is the node's startup path, including consistency checks, the durable unwind - // marker and Merkle's walk_all_changed_branch_children recovery mode. - let factory = runtime.handle().block_on( - context.create_provider_factory_with_recovery::( - OverlayManager::new(runtime.state_trie_overlay_worker_pool()), - None, - &[], - |factory| { - let checkpoint = - factory.provider()?.get_stage_checkpoint(StageId::Finish)?.unwrap_or_default(); - let partial = checkpoint - .finish_stage_checkpoint() - .and_then(|finish| finish.partial_state_trie()); - println!( - "{}", - json!({"recovery": "frontiers", "finish": checkpoint.block_number, - "partial_state_trie": partial}) - ); - Ok(()) - }, - ), - )?; - reth_db_common::init::init_genesis(&factory)?; - if let Some(path) = config.init_socket.take() { - ensure!( - factory - .provider()? - .get_stage_checkpoint(StageId::Finish)? - .unwrap_or_default() - .block_number == - 0, - "warm initialization requires empty state" - ); - wait_for_execution(config, &path)?; - } - let input = config - .input - .take() - .unwrap_or_else(|| Input::generate(config.seed, config.blocks, config.updates)); - input.validate()?; - let input_blocks = - input.operations.split_inclusive(|op| matches!(op, Operation::Commit)).collect::>(); - let mut total_blocks = input_blocks.len() as u64; - ensure!( - config.expected_roots.as_ref().is_none_or(|roots| roots.len() == input_blocks.len()), - "expected root count does not match input" - ); - let overrides_path = datadir.join("input-overrides.json"); - #[derive(Default, Serialize, Deserialize)] - struct Overrides { - applied: u64, - blocks: BTreeMap>, - total_blocks: u64, - } - #[derive(Deserialize)] - struct Injection { - id: u64, - input: Input, - } - let mut overrides: Overrides = if overrides_path.exists() { - serde_json::from_slice(&fs::read(&overrides_path)?)? - } else { - Overrides::default() - }; - if overrides.total_blocks != 0 { - total_blocks = overrides.total_blocks; - // Saved continuations no longer match roots precomputed for the original input. - config.expected_roots = None; - println!("{}", json!({"input": "restored", "blocks": total_blocks})); - } - let journal_path = datadir.join("blocks.jsonl"); - let mut journal = BTreeMap::::new(); - if journal_path.exists() { - for line in - fs::read(&journal_path)?.split(|byte| *byte == b'\n').filter(|line| !line.is_empty()) - { - let block: JournalBlock = match serde_json::from_slice(line) { - Ok(block) => block, - // SIGKILL may interrupt the final append. Each new append starts on a new line. - Err(error) if error.is_eof() => continue, - Err(error) => return Err(error.into()), - }; - ensure!((1..=total_blocks).contains(&block.number), "journal block out of range"); - if let Some(previous) = journal.get(&block.number) { - ensure!(previous == &block, "conflicting journal block {}", block.number); - } else { - journal.insert(block.number, block); - } - } - } - let mut journal_file = OpenOptions::new().create(true).append(true).open(&journal_path)?; - - let manager = factory.overlay_manager(); - let base = - factory.provider()?.get_stage_checkpoint(StageId::Finish)?.unwrap_or_default().block_number; - let mut parent = factory - .provider()? - .sealed_header(base)? - .ok_or_else(|| eyre::eyre!("missing header {base}"))?; - let base_hash = parent.hash(); - let strategy = DefaultStateRootStrategy::default(); - let mut state = Model::default(); - for number in 1..=base { - let block = - journal.get(&number).ok_or_else(|| eyre::eyre!("missing journal block {number}"))?; - for operation in &block.operations { - if let Operation::UpdateBatch { updates } = operation { - for update in updates { - update.apply_to_model(&mut state); - } - } - } - } - if base > 0 { - verify(&factory, &state, base, base)?; - } - let recovered_root = root(&state); - ensure!(recovered_root == parent.state_root(), "recovered root at {base}"); - println!("{}", json!({"recovery": "complete", "head": base, "root": recovered_root})); - let mut blocks = Vec::>::new(); - // Replay journal updates instead of retaining a full model for every block. - let mut persisted_state = state.clone(); - let timeout = Duration::from_secs(config.timeout_secs); - let (request_tx, request_rx) = mpsc::sync_channel::(1); - let (ack_tx, ack_rx) = mpsc::sync_channel(1); - let writer_factory = factory.clone(); - let writer = std::thread::spawn(move || { - while let Ok(input) = request_rx.recv() { - let result = (|| -> Result<()> { - let provider = writer_factory.provider_rw()?; - provider.save_blocks(&input)?; - provider.commit()?; - Ok(()) - })(); - if ack_tx.send(result).is_err() { - break; - } - } - }); - let mut db_tip = base; - let mut state_tip = base; - let mut commits = 0; - let mut partial_commits = 0; - let mut update_batches = 0; - let mut hint_batches = 0; - let mut pending_save = None; - let mut prune_requested = false; - let mut shutdown = false; - let mut waiting = false; - let mut seen_input_file = None; - loop { - let head = base + blocks.len() as u64; - if head < 1024 && - journal.range((head + 1)..).next().is_none() && - let Some((path, file_stamp)) = config.input_file.as_ref().and_then(|path| { - fs::metadata(path).ok().map(|metadata| { - ( - path, - (metadata.ino(), metadata.mtime(), metadata.mtime_nsec(), metadata.len()), - ) - }) - }) && - seen_input_file != Some(file_stamp) - { - let injection: Injection = serde_json::from_slice(&fs::read(path)?)?; - if injection.id != overrides.applied { - injection.input.validate()?; - let injected = injection.input.blocks(); - let count = injected.len().min((1024 - head) as usize); - ensure!(count > 0, "continuation exceeds the 1024-block limit"); - overrides.blocks.retain(|n, _| *n < (head + 1)); - for (i, block) in injected.into_iter().take(count).enumerate() { - overrides.blocks.insert((head + 1) + i as u64, block); - } - total_blocks = head + count as u64; - overrides.total_blocks = total_blocks; - overrides.applied = injection.id; - config.expected_roots = None; - shutdown = false; - waiting = false; - let tmp = overrides_path.with_extension("tmp"); - fs::write(&tmp, serde_json::to_vec(&overrides)?)?; - fs::rename(tmp, &overrides_path)?; - println!("{}", json!({"input": "injected", "block": (head + 1), "blocks": count})); - } - seen_input_file = Some(file_stamp); - } - if let Some((next_db, next_state)) = pending_save { - let result = if shutdown || - head.saturating_sub(state_tip).saturating_sub(buffer) >= backpressure - { - Some(ack_rx.recv_timeout(timeout)?) - } else { - match ack_rx.try_recv() { - Ok(result) => Some(result), - Err(mpsc::TryRecvError::Empty) => None, - Err(mpsc::TryRecvError::Disconnected) => { - eyre::bail!("persistence worker disconnected") - } - } - }; - if let Some(result) = result { - result?; - pending_save = None; - for number in (db_tip + 1)..=next_db { - let block = journal - .get(&number) - .ok_or_else(|| eyre::eyre!("missing journal block {number}"))?; - for operation in &block.operations { - if let Operation::UpdateBatch { updates } = operation { - for update in updates { - update.apply_to_model(&mut persisted_state); - } - } - } - } - db_tip = next_db; - state_tip = next_state; - commits += 1; - partial_commits += u64::from(db_tip != state_tip); - manager.evict_cached_changesets( - db_tip.saturating_sub(CHANGESET_CACHE_RETENTION_BLOCKS), - ); - verify(&factory, &persisted_state, db_tip, state_tip)?; - manager.remove_blocks( - blocks - .iter() - .filter(|block| block.recovered_block().number() <= state_tip) - .map(|block| block.recovered_block().hash()), - ); - let anchor = if state_tip == base { - base_hash - } else { - blocks[(state_tip - base) as usize - 1].recovered_block().hash() - }; - manager.precompute_execution_overlay(parent.hash(), anchor); - prune_requested = true; - // The controller may SIGKILL this process after this event. - #[cfg(feature = "inline-coverage")] - unsafe { - bedrock_coverage_flush(); - } - println!( - "{}", - json!({"persistence": "complete", "head": head, - "db_tip": db_tip, "state_tip": state_tip}) - ); - continue; - } - } - if pending_save.is_none() { - // Match EngineApiTreeHandler::get_save_blocks_input's Threshold and Head targets. - let next_db = if shutdown { head } else { head.saturating_sub(buffer) }; - if shutdown || - (head.saturating_sub(state_tip) > config.persistence_threshold && - next_db > db_tip) - { - let next_state = - if shutdown { head } else { next_db.saturating_sub(masking).max(state_tip) }; - if next_db != db_tip || next_state != state_tip { - println!( - "{}", - json!({"persistence": "start", "head": head, - "db_tip": db_tip, "state_tip": state_tip, - "next_db_tip": next_db, "next_state_tip": next_state, - "shutdown": shutdown}) - ); - request_tx.send(SaveBlocksInput::new( - blocks[(state_tip - base) as usize..(next_db - base) as usize].to_vec(), - db_tip, - state_tip, - next_db, - next_state, - ))?; - pending_save = Some((next_db, next_state)); - continue; - } - } - } - if head == total_blocks { - if shutdown { - if head < 1024 && - let Some(path) = &config.input_file && - !path.with_extension("finish").exists() - { - if !waiting { - println!("{}", json!({"input": "waiting", "head": head})); - waiting = true; - } - std::thread::sleep(Duration::from_millis(10)); - continue; - } - break; - } - // Let the last block trigger normal threshold persistence before shutdown. - shutdown = true; - continue; - } - let number = head + 1; - let block = match journal.entry(number) { - Entry::Occupied(entry) => entry.into_mut(), - Entry::Vacant(entry) => { - let block = overrides - .blocks - .get(&number) - .map_or_else(|| input_blocks[number as usize - 1].to_vec(), Clone::clone); - let journal_block = JournalBlock { number, operations: block }; - let record = serde_json::to_vec(&journal_block)?; - journal_file.write_all(b"\n")?; - journal_file.write_all(&record)?; - journal_file.write_all(b"\n")?; - entry.insert(journal_block) - } - }; - let operations = block.operations.clone(); - let updates_input = operations - .iter() - .flat_map(|op| match op { - Operation::UpdateBatch { updates } => updates.as_slice(), - _ => &[], - }) - .cloned() - .collect::>(); - let mut accounts = state.keys().copied().collect::>(); - for op in &updates_input { - match op { - model::Update::Account { address, .. } => { - accounts.insert(*address); - } - model::Update::Storage { address, .. } => ensure!( - accounts.contains(address), - "storage before account creation: {address}" - ), - } - } - let (bundle, updates) = apply(&mut state, number, &updates_input); - // The full operations are in the journal before production tasks run. - #[cfg(feature = "inline-coverage")] - unsafe { - bedrock_coverage_flush(); - } - println!( - "{}", - json!({"block": number, "seed": config.seed, "db_tip": db_tip, - "state_tip": state_tip, "operations": operations.len()}) - ); - let factory_view = OverlayStateProviderFactory::new( - factory.clone(), - manager.overlay_builder(parent.hash()), - ); - let retained = std::mem::take(&mut prune_requested).then(|| { - blocks - .iter() - .filter(|block| block.recovered_block().number() > state_tip) - .rev() - .cloned() - .collect() - }); - let (mut handle, pending) = strategy.spawn_state_root( - &runtime, - factory_view, - StateRootTaskOptions { - parent_header: parent.clone(), - preserved_sparse_trie: manager.take_sparse_trie(), - transaction_count: None, - gas_used: None, - config: &tree_config, - pending_sparse_trie_prune_blocks: retained, - }, - ); - let stream = handle.take_hashed_update_stream(); - let hints = handle.take_hint_stream(); - let written = updates - .iter() - .flat_map(|update| { - update - .accounts - .keys() - .chain(update.storages.keys()) - .map(|address| (*address, None)) - .chain(update.storages.iter().flat_map(|(address, storage)| { - storage.storage.keys().map(move |slot| (*address, Some(*slot))) - })) - }) - .collect::>(); - let mut hinted = BTreeSet::new(); - let mut updates = updates.into_iter(); - for operation in &operations { - match operation { - Operation::PrewarmBatch { accounts, storages } => { - let mut hint = StateAccessHint::default(); - for address in accounts { - let address = keccak256(address); - hint.accounts.push(address); - hinted.insert((address, None)); - } - for storage in storages { - let address = keccak256(storage.address); - let slot = keccak256(B256::from(storage.slot)); - hint.storages.entry(address).or_default().push(slot); - hinted.insert((address, Some(slot))); - } - hints.on_access_hint(hint); - hint_batches += 1; - } - Operation::UpdateBatch { updates: batch_updates } => { - let mut batch = HashedPostState::default(); - for update in updates.by_ref().take(batch_updates.len()) { - batch.extend(update); - } - stream.on_hashed_state_update(batch); - update_batches += 1; - } - Operation::Commit => {} - } - } - println!( - "{}", - json!({"block_number": number, "hint_coverage": { - "written": written.len(), "overlap": written.intersection(&hinted).count(), - "unhinted": written.difference(&hinted).count(), - "read_only": hinted.difference(&written).count()}}) - ); - drop(hints); - stream.finish(); - let outcome = handle.take_state_root_rx().recv_timeout(timeout)??; - drop(handle); - let expected_root = config - .expected_roots - .as_ref() - .and_then(|roots| roots.get(number as usize - 1)) - .copied() - .unwrap_or_else(|| root(&state)); - ensure!( - outcome.state_root == expected_root, - "seed {} block {number}: expected {}, actual {}", - config.seed, - expected_root, - outcome.state_root - ); - let block = RecoveredBlock::new_unhashed( - Block { - header: Header { - parent_hash: parent.hash(), - number, - state_root: outcome.state_root, - timestamp: number, - ..Default::default() - }, - ..Default::default() - }, - Vec::new(), - ); - parent = SealedHeader::new_unhashed(block.header().clone()); - let executed = ExecutedBlock::new( - Arc::new(block), - Arc::new(BlockExecutionOutput { - state: bundle, - result: BlockExecutionResult { - receipts: Vec::new(), - requests: Default::default(), - gas_used: 0, - blob_gas_used: 0, - }, - }), - SortedTrieData { - hashed_state: Arc::new(match Arc::try_unwrap(outcome.hashed_state) { - Ok(state) => state.into_sorted(), - Err(state) => state.clone_into_sorted(), - }), - trie_updates: Arc::new(match Arc::try_unwrap(outcome.trie_updates) { - Ok(updates) => updates.into_sorted(), - Err(updates) => updates.clone_into_sorted(), - }), - }, - ); - manager.insert_block(executed.clone()); - publish_sparse_trie(&pending, manager, parent.hash(), outcome.state_root); - blocks.push(executed); - } - drop(request_tx); - writer.join().map_err(|_| eyre::eyre!("persistence worker panicked"))?; - // Synchronize final trie completion before taking the metrics snapshot. - if let Some(trie) = manager.take_sparse_trie() { - ensure!(trie.into_trie_for(parent.hash())?.is_some(), "missing final reusable trie"); - } - // Root delivery precedes worker teardown. Drain their queues so the last database handle - // closes before process exit destroys RocksDB's global scheduler. - for name in ["sparse-trie", "trie-hashing", "storage-workers", "account-workers", "drop"] { - runtime.spawn_blocking_named(name, || {}).get(); - } - let metrics = metric_counts(snapshotter); - println!( - "{}", - json!({"completed": true, "seed": config.seed, "blocks": total_blocks, - "commits": commits, "partial_commits": partial_commits, - "update_batches": update_batches, "hint_batches": hint_batches, - "metrics": metrics}) - ); - Ok(()) -} - -fn main() -> Result<()> { - let mut argv = std::env::args().skip(1); - let command = argv.next(); - let text = match command.as_deref() { - Some("mutation-operators") => { - println!( - "{}", - json!([ - {"name": "account.address", "selector": {"path": ["operations", "*", "updates", "*", "Account", "address"]}}, - {"name": "account.balance", "selector": {"path": ["operations", "*", "updates", "*", "Account", "account", "balance"]}}, - {"name": "account.nonce", "selector": {"path": ["operations", "*", "updates", "*", "Account", "account", "nonce"]}}, - {"name": "account.bytecode_hash", "selector": {"path": ["operations", "*", "updates", "*", "Account", "account", "bytecode_hash"]}}, - {"name": "storage.address", "selector": {"path": ["operations", "*", "updates", "*", "Storage", "address"]}}, - {"name": "storage.slot", "selector": {"path": ["operations", "*", "updates", "*", "Storage", "slot"]}}, - {"name": "storage.value", "selector": {"path": ["operations", "*", "updates", "*", "Storage", "value"]}}, - {"name": "prewarm.account", "selector": {"path": ["operations", "*", "accounts", "*"]}}, - {"name": "prewarm.storage.address", "selector": {"path": ["operations", "*", "storages", "*", "address"]}}, - {"name": "prewarm.storage.slot", "selector": {"path": ["operations", "*", "storages", "*", "slot"]}}, - {"name": "update.duplicate", "selector": {"path": ["operations", "*", "updates", "*"]}}, - {"name": "update.remove", "selector": {"path": ["operations", "*", "updates", "*"]}}, - {"name": "update.crossover", "donor": true, "selector": {"path": ["operations", "*", "updates", "*"]}}, - {"name": "batch.duplicate", "selector": {"path": ["operations", "*"], "kinds": ["update_batch", "prewarm_batch"]}}, - {"name": "batch.remove", "selector": {"path": ["operations", "*"], "kinds": ["update_batch", "prewarm_batch"]}}, - {"name": "batch.crossover", "donor": true, "selector": {"path": ["operations", "*"], "kinds": ["update_batch", "prewarm_batch"]}}, - {"name": "commit.insert", "selector": {"path": ["operations", "*"], "kinds": ["update_batch"]}}, - {"name": "commit.remove", "selector": {"path": ["operations", "*"], "kinds": ["commit"], "exclude_last": true}} - ]) - ); - return Ok(()); - } - Some("mutation-sites") => { - let mut request = Vec::new(); - std::io::stdin().read_to_end(&mut request)?; - let mut value: serde_json::Value = serde_json::from_slice(&request)?; - let input: Input = match std::mem::take(&mut value["input"]) { - serde_json::Value::String(text) => serde_json::from_str(&text)?, - input => serde_json::from_value(input)?, - }; - input.validate()?; - let operator = value["operator"].as_str(); - let sites = input.mutation_sites(operator); - println!("{}", serde_json::to_string(&sites)?); - return Ok(()); - } - Some("observe") => { - let mut request = Vec::new(); - std::io::stdin().read_to_end(&mut request)?; - let value: serde_json::Value = serde_json::from_slice(&request)?; - let decisions = - value.get("decisions").ok_or_else(|| eyre::eyre!("missing decisions"))?; - let changed = value - .get("original") - .is_some_and(|original| !original.is_null() && original != decisions); - println!( - "{}", - json!({ - "decisions": decisions, - "decision_diff": if changed { vec!["recorded decisions differ"] } else { vec![] }, - "moments": [], "events_origin_ns": null - }) - ); - return Ok(()); - } - Some("generate-input" | "generate-suffix" | "mutate-input" | "validate-input") => { - let mut request = Vec::new(); - std::io::stdin().read_to_end(&mut request)?; - let mut value: serde_json::Value = serde_json::from_slice(&request)?; - let contract = value["input"].is_string(); - let input = match command.as_deref() { - Some("generate-input" | "generate-suffix") => { - let config: Config = serde_json::from_value(value)?; - ensure!( - (1..=1024).contains(&config.blocks) && (1..=4096).contains(&config.updates), - "invalid input size" - ); - let mut input = Input::generate(config.seed, config.blocks, config.updates); - if command.as_deref() == Some("generate-suffix") { - let accounts = input - .operations - .iter() - .filter_map(|op| match op { - Operation::UpdateBatch { updates } => Some(updates.iter()), - _ => None, - }) - .flatten() - .filter_map(|update| match update { - model::Update::Account { address, .. } => Some(*address), - _ => None, - }) - .collect::>() - .into_iter() - .map(|address| model::Update::Account { - address, - account: reth_primitives_traits::Account { - nonce: 1, - ..Default::default() - }, - }) - .collect::>(); - let Operation::UpdateBatch { updates } = input - .operations - .iter_mut() - .find(|op| matches!(op, Operation::UpdateBatch { .. })) - .expect("generated update batch") - else { - unreachable!() - }; - updates.splice(0..0, accounts); - } - input - } - _ => { - let mut input: Input = match std::mem::take(&mut value["input"]) { - serde_json::Value::String(text) => serde_json::from_str(&text)?, - input => serde_json::from_value(input)?, - }; - if command.as_deref() == Some("mutate-input") { - let seed = value["seed"] - .as_u64() - .ok_or_else(|| eyre::eyre!("missing mutation seed"))?; - if let Some(operator) = value["operator"].as_str().map(str::to_owned) { - let path: Vec = serde_json::from_value(value["path"].clone())?; - let donor: Option = match std::mem::take(&mut value["donor"]) { - serde_json::Value::String(text) => { - Some(serde_json::from_str(&text)?) - } - donor => serde_json::from_value(donor)?, - }; - let donor_path: Vec = if donor.is_some() { - serde_json::from_value(value["donor_path"].clone())? - } else { - Vec::new() - }; - input.mutate_at( - &operator, - &path, - seed, - donor.as_ref().map(|donor| (donor, donor_path.as_slice())), - )?; - } else { - input.mutate(seed)?; - } - } - input - } - }; - input.validate()?; - if command.as_deref() == Some("validate-input") { - let mut state = Model::default(); - let expected_roots = input - .operations - .split_inclusive(|op| matches!(op, Operation::Commit)) - .map(|block| { - for op in block { - if let Operation::UpdateBatch { updates } = op { - for update in updates { - update.apply_to_model(&mut state); - } - } - } - root(&state) - }) - .collect::>(); - println!( - "{}", - json!({"blocks": expected_roots.len(), "expected_roots": expected_roots}) - ); - return Ok(()); - } - let output = serde_json::to_string(&input)?; - if command.as_deref() == Some("mutate-input") && contract { - println!("{}", serde_json::to_string(&output)?); - } else { - println!("{output}"); - } - return Ok(()); - } - Some(hook @ ("parse-args" | "config" | "seed-inputs" | "suffix-seed")) => { - let controller = std::env::current_exe()?.with_file_name("controller.py"); - return Err(std::process::Command::new(controller).arg(hook).args(argv).exec().into()); - } - Some("-") => std::io::read_to_string(std::io::stdin())?, - Some(path) if path.starts_with('@') => fs::read_to_string(&path[1..])?, - Some(text) => text.to_string(), - None => "{}".into(), - }; - let mut config: Config = serde_json::from_str(&text)?; - drop(text); - if let Some(input) = &config.input { - input.validate()?; - } - let recorder = DebuggingRecorder::new(); - let snapshotter = recorder.snapshotter(); - recorder.install()?; - run(&mut config, &snapshotter) - .wrap_err_with(|| format!("reth-trie-dst seed={} blocks={}", config.seed, config.blocks)) -} - -// Only these settings affect the runtime, tree configuration and storage initialized above. -fn validate_initialization(initial: &Config, execution: &Config) -> Result<()> { - ensure!( - initial.proof_workers == execution.proof_workers && - initial.persistence_threshold == execution.persistence_threshold && - initial.num_state_masking_blocks == execution.num_state_masking_blocks && - initial.memory_block_buffer_target == execution.memory_block_buffer_target && - initial.datadir == execution.datadir && - execution.init_socket.is_none(), - "execution changes initialized configuration" - ); - ensure!((1..=1024).contains(&execution.blocks), "blocks must be 1..=1024"); - ensure!((1..=4096).contains(&execution.updates), "updates must be 1..=4096"); - ensure!((1..=3600).contains(&execution.timeout_secs), "timeout_secs must be 1..=3600"); - Ok(()) -} - -fn wait_for_execution(config: &mut Config, path: &std::path::Path) -> Result<()> { - let socket = UnixDatagram::bind(path)?; - let deadline = Instant::now() + Duration::from_secs(3600); - let mut message = [0u8; 4096]; - loop { - socket.set_read_timeout(Some( - deadline - .checked_duration_since(Instant::now()) - .ok_or_else(|| eyre::eyre!("initialization barrier timed out"))?, - ))?; - let (size, peer) = socket.recv_from(&mut message)?; - let peer = peer.as_pathname().ok_or_else(|| eyre::eyre!("unnamed barrier peer"))?; - if &message[..size] == b"ready" { - socket.send_to(b"initialized", peer)?; - continue; - } - let execution: Config = - serde_json::from_slice(&fs::read(std::str::from_utf8(&message[..size])?)?)?; - validate_initialization(config, &execution)?; - *config = execution; - socket.send_to(b"started", peer)?; - fs::remove_file(path)?; - return Ok(()); - } -} diff --git a/contrib/trie-dst/crates/reth-trie-dst/src/model.rs b/contrib/trie-dst/crates/reth-trie-dst/src/model.rs deleted file mode 100644 index 010576557ff..00000000000 --- a/contrib/trie-dst/crates/reth-trie-dst/src/model.rs +++ /dev/null @@ -1,220 +0,0 @@ -//! Reference state and generated execution outcomes, independent of trie-task output. - -use alloy_primitives::{ - keccak256, - map::{HashMap, U256Map}, - Address, B256, KECCAK256_EMPTY, U256, -}; -use alloy_trie::{HashBuilder, Nibbles, TrieAccount}; -use evm2::evm::AccountInfo; -use rand::{rngs::StdRng, Rng}; -use reth_primitives_traits::Account; -use reth_trie::{bundle_state::BundleState, HashedPostState, HashedStorage}; -use serde::{Deserialize, Serialize}; -use std::collections::{BTreeMap, BTreeSet}; - -#[derive(Clone, Debug, Default)] -pub(crate) struct ModelAccount { - pub account: Account, - pub storage: BTreeMap, - hashed_storage: BTreeMap>, - pub touched_slots: BTreeSet, -} - -pub(crate) type Model = BTreeMap; - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) enum Update { - Account { address: Address, account: Account }, - Storage { address: Address, slot: U256, value: U256 }, -} - -impl Update { - pub(crate) fn apply_to_model(&self, state: &mut Model) -> (Address, Option) { - match self { - Self::Account { address, account } => { - state.entry(*address).or_default().account = account.to_owned(); - (*address, None) - } - Self::Storage { address, slot, value } => { - let entry = state.get_mut(address).expect("storage account exists"); - entry.touched_slots.insert(*slot); - let hashed_slot = keccak256(B256::from(*slot)); - if value.is_zero() { - entry.storage.remove(slot); - entry.hashed_storage.remove(&hashed_slot); - } else { - entry.storage.insert(*slot, *value); - entry.hashed_storage.insert(hashed_slot, alloy_rlp::encode(value)); - } - (*address, Some(hashed_slot)) - } - } - } - - fn apply(&self, state: &mut Model) -> (Address, HashedPostState) { - let (address, hashed_slot) = self.apply_to_model(state); - let mut delta = HashedPostState::default(); - match self { - Self::Account { account, .. } => { - delta.accounts.insert(keccak256(address), Some(account.to_owned())); - } - Self::Storage { value, .. } => { - delta.storages.insert( - keccak256(address), - HashedStorage { - storage: HashMap::from_iter([(hashed_slot.expect("storage slot"), *value)]), - }, - ); - } - } - (address, delta) - } -} - -pub(crate) fn random_address(rng: &mut impl Rng) -> Address { - Address::from_word(B256::from(U256::from(rng.random::()))) -} - -pub(crate) fn random_slot(rng: &mut impl Rng) -> U256 { - U256::from(rng.random::()) -} - -#[allow(clippy::clone_on_copy, reason = "Account is not Copy with account-ext enabled.")] -pub(crate) fn generate(rng: &mut StdRng, state: &Model, updates: usize) -> Vec { - let mut state = state.clone(); - let mut addresses = state.keys().copied().collect::>(); - (0..updates) - .map(|_| { - // Bound the active account set while sampling new keys across all 32 bits. - let address = - if !addresses.is_empty() && (addresses.len() >= 16 || rng.random_ratio(3, 4)) { - addresses[rng.random_range(0..addresses.len())] - } else { - random_address(rng) - }; - let operation = if !state.contains_key(&address) || rng.random_ratio(1, 4) { - let mut account = - state.get(&address).map(|entry| entry.account.clone()).unwrap_or_default(); - account.nonce += 1; - account.balance = U256::from(rng.random_range(1..=1_000_000u64)); - Update::Account { address, account } - } else { - let known_slots = &state[&address].touched_slots; - let slot = if !known_slots.is_empty() && rng.random_bool(0.5) { - *known_slots.iter().nth(rng.random_range(0..known_slots.len())).unwrap() - } else { - random_slot(rng) - }; - let value = if rng.random_ratio(1, 3) { - U256::ZERO - } else { - U256::from(rng.random::()) - }; - Update::Storage { address, slot, value } - }; - if !state.contains_key(&address) { - addresses.push(address); - } - operation.apply(&mut state); - operation - }) - .collect() -} - -#[allow(clippy::clone_on_copy, reason = "Account is not Copy with account-ext enabled.")] -pub(crate) fn apply( - state: &mut Model, - number: u64, - operations: &[Update], -) -> (BundleState, Vec) { - let mut before = BTreeMap::, BTreeMap)>::new(); - let mut touched = BTreeMap::>::new(); - let mut stream = Vec::new(); - for operation in operations { - let address = match operation { - Update::Account { address, .. } | Update::Storage { address, .. } => *address, - }; - let (_, old_slots) = before.entry(address).or_insert_with(|| { - (state.get(&address).map(|entry| entry.account.clone()), BTreeMap::new()) - }); - if let Update::Storage { slot, .. } = operation { - old_slots.entry(*slot).or_insert_with(|| { - state - .get(&address) - .and_then(|entry| entry.storage.get(slot)) - .copied() - .unwrap_or_default() - }); - } - let (address, delta) = operation.apply(state); - let slots = touched.entry(address).or_default(); - if let Update::Storage { slot, .. } = operation { - slots.insert(*slot); - } - stream.push(delta); - } - - let mut bundle = BundleState::builder(number..=number); - for (address, slots) in touched { - let (old_account, old_slots) = &before[&address]; - let current = &state[&address]; - let info = AccountInfo { - balance: current.account.balance, - nonce: current.account.nonce, - code_hash: current.account.bytecode_hash.unwrap_or(KECCAK256_EMPTY), - ..Default::default() - }; - bundle = bundle.state_present_account_info(address, info); - let old_info = old_account.as_ref().map(|account| AccountInfo { - balance: account.balance, - nonce: account.nonce, - code_hash: account.bytecode_hash.unwrap_or(KECCAK256_EMPTY), - ..Default::default() - }); - if let Some(info) = &old_info { - bundle = bundle.state_original_account_info(address, info.clone()); - } - bundle = bundle.revert_account_info(number, address, Some(old_info)); - let storage = slots - .into_iter() - .map(|slot| { - let original = old_slots[&slot]; - let present = current.storage.get(&slot).copied().unwrap_or_default(); - (slot, (original, present)) - }) - .collect::>(); - bundle = bundle - .revert_storage( - number, - address, - storage.iter().map(|(slot, (original, _))| (*slot, *original)).collect(), - ) - .state_storage(address, storage); - } - (bundle.build(), stream) -} - -pub(crate) fn storage_root(account: &ModelAccount) -> B256 { - let mut builder = HashBuilder::default(); - for (key, value) in &account.hashed_storage { - builder.add_leaf(Nibbles::unpack(*key), value); - } - builder.root() -} - -#[allow(clippy::clone_on_copy, reason = "Account is not Copy with account-ext enabled.")] -pub(crate) fn root(state: &Model) -> B256 { - let leaves = state - .iter() - .map(|(address, entry)| { - let account: TrieAccount = entry.account.clone().into_trie_account(storage_root(entry)); - (keccak256(address), alloy_rlp::encode(account)) - }) - .collect::>(); - let mut builder = HashBuilder::default(); - for (key, value) in leaves { - builder.add_leaf(Nibbles::unpack(key), &value); - } - builder.root() -} diff --git a/contrib/trie-dst/workload/bedrock.yaml b/contrib/trie-dst/workload/bedrock.yaml deleted file mode 100644 index 61350ad1e82..00000000000 --- a/contrib/trie-dst/workload/bedrock.yaml +++ /dev/null @@ -1,16 +0,0 @@ -prepare: [./prepare.sh] -compose: compose.yaml -planner: reth-trie-dst -command: podman exec reth-trie-dst /usr/local/bin/controller.py -fuzz_args: [--max-snaps=0, --run-secs=1] -memory_mb: 1536 -workload_args: - cold_start: false # Snapshot a live initialized worker; true retains startup coverage. - blocks: 64 - updates: 64 - proof_workers: 2 - timeout_secs: 60 - persistence_threshold: 50 - num_state_masking_blocks: 30 - memory_block_buffer_target: null # Reth defaults to min(5, persistence_threshold). - crashes: 1 # SIGKILL/restart the worker, retaining its database and block journal. diff --git a/contrib/trie-dst/workload/compose.yaml b/contrib/trie-dst/workload/compose.yaml deleted file mode 100644 index f8e2ed58bbd..00000000000 --- a/contrib/trie-dst/workload/compose.yaml +++ /dev/null @@ -1,11 +0,0 @@ -services: - reth-trie-dst: - image: bedrock/reth-trie-dst:latest - pull_policy: never - container_name: reth-trie-dst - privileged: true - environment: - BEDROCK_COVERAGE_DIR: /bedrock/coverage - volumes: - - /bedrock/in:/bedrock/in:ro - - /bedrock/out:/bedrock/out diff --git a/contrib/trie-dst/workload/controller.py b/contrib/trie-dst/workload/controller.py deleted file mode 100755 index ae26fbe47f9..00000000000 --- a/contrib/trie-dst/workload/controller.py +++ /dev/null @@ -1,507 +0,0 @@ -#!/usr/bin/env python3 -"""Bedrock workload hooks; native state generation and checking live in Rust.""" - -import json -import os -import random -import shutil -import socket -import tempfile -from pathlib import Path -import signal -import subprocess -import sys -import time - -# Persistence defaults match TreeConfig and EngineArgs in this Reth revision. -DEFAULTS = {"blocks": 64, "updates": 64, "proof_workers": 2, "timeout_secs": 60, - "persistence_threshold": 50, "num_state_masking_blocks": 30, - "memory_block_buffer_target": None, "crashes": 1, "cold_start": False} -LIMITS = {"crashes": (0, 16), "blocks": (1, 1024), "updates": (1, 4096), "proof_workers": (1, 16), - "timeout_secs": (1, 3600), "persistence_threshold": (0, 2**64 - 2), - "num_state_masking_blocks": (0, 2**64 - 1), - "memory_block_buffer_target": (0, 2**64 - 1)} -STATE = Path("/tmp/reth-trie-dst") -ASSERTIONS = Path("/bedrock/assertions/reth-trie-dst.jsonl") - - -def arguments(values): - unknown = values.keys() - DEFAULTS.keys() - if unknown: - raise ValueError(f"unknown arguments: {sorted(unknown)}") - result = DEFAULTS | values - if type(result["cold_start"]) is not bool: - raise ValueError("cold_start must be a boolean") - for key, (low, high) in LIMITS.items(): - if key == "memory_block_buffer_target" and result[key] is None: - continue - if type(result[key]) is not int or not low <= result[key] <= high: - raise ValueError(f"{key} must be an integer in {low}..={high}") - if result["crashes"] > result["blocks"]: - raise ValueError("crashes must not exceed blocks") - threshold = result["persistence_threshold"] - buffer = result["memory_block_buffer_target"] - if buffer is None: - buffer = min(5, threshold) - masking = result["num_state_masking_blocks"] if threshold else 0 - if buffer > threshold: - raise ValueError("buffer must not exceed persistence threshold") - if masking and masking + buffer >= threshold: - raise ValueError("masking blocks + buffer must be less than persistence threshold") - return result - - -def assertion(ok, detail, signature="trie/run/checked", kind="Always"): - ASSERTIONS.parent.mkdir(parents=True, exist_ok=True) - with ASSERTIONS.open("a") as output: - record = {"condition": {"Bool": ok}, "result": ok, - "message": f"{signature}: {detail}", - "location": {"file": "controller.py", "line": 35, "column": 1}, - "timestamp_unix_nano": time.time_ns()} - output.write(json.dumps({kind: record}) + "\n") - - -def collect(destination, progress=False): - offsets_path = Path("/bedrock/in/assertion-offsets.json") - cursor = STATE / "progress-offsets.json" - offsets = json.loads(offsets_path.read_text()) if offsets_path.exists() else {} - if progress and cursor.exists(): - offsets = json.loads(cursor.read_text()) - records = [] - for path in sorted(Path("/bedrock/assertions").glob("*.jsonl")): - with path.open("rb") as source: - start = min(offsets.get(path.name, 0), path.stat().st_size) - source.seek(start) - data = source.read() - complete = data.rfind(b"\n") + 1 - records.extend(data[:complete].decode().splitlines()) - offsets[path.name] = start + complete - destination.parent.mkdir(parents=True, exist_ok=True) - records.sort(key=lambda line: next(iter(json.loads(line).values())).get("timestamp_unix_nano", 0)) - destination.write_text("".join(line + "\n" for line in records)) - if progress: - cursor.write_text(json.dumps(offsets)) - else: - for name in ("run.log", "controller.log", "done.json", "stdout.log", "init.log", "warm-ready.json", "data/blocks.jsonl"): - source = STATE / name - if source.exists(): - shutil.copyfile(source, destination.parent / ("reth-trie-dst-" + source.name)) - - -def native(hook, request): - binary = Path(__file__).with_name("reth-trie-dst") - # Host input hooks need no coverage; the guest opts in through compose.yaml. - env = os.environ.copy() - env.pop("BEDROCK_COVERAGE_DIR", None) - result = subprocess.run([str(binary), hook], input=json.dumps(request), text=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, env=env) - return json.loads(result.stdout) - - -def alive(pid): - try: - # kill(0) also succeeds for zombies; never report one as ready. - return Path(f"/proc/{pid}/stat").read_text().split(") ", 1)[1][0] != "Z" - except FileNotFoundError: - return False - - -def barrier(message, expected, timeout=1, process=None): - with tempfile.TemporaryDirectory(dir=STATE) as directory: - with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as connection: - connection.bind(str(Path(directory) / "reply")) - deadline = time.monotonic() + timeout - connection.sendto(message.encode(), str(STATE / "init.sock")) - while True: - connection.settimeout(max(0.001, min(0.1, deadline - time.monotonic()))) - try: - response = connection.recv(128) - except TimeoutError: - if process is not None and (code := process.poll()) is not None: - raise RuntimeError(f"warm worker exited during handshake: exit={code}") - if time.monotonic() >= deadline: - raise - continue - if response != expected: - raise RuntimeError("invalid initialization barrier response") - return - - -class WarmProcess: - """Adopt a guest process without retaining a host-side pipe or Popen object.""" - - def __init__(self): - self.pid = int((STATE / "warm.pid").read_text()) - - def poll(self): - status = STATE / "warm.exit" - if status.exists(): - return int(status.read_text()) - if not alive(self.pid) and not alive(int((STATE / "launcher.pid").read_text())): - return -signal.SIGKILL - return None - - def send_signal(self, value): - os.kill(self.pid, value) - - def kill(self): - self.send_signal(signal.SIGKILL) - - def wait(self, timeout): - deadline = time.monotonic() + timeout - while self.poll() is None: - if time.monotonic() >= deadline: - raise TimeoutError("warm worker did not exit") - time.sleep(0.01) - return self.poll() - - -def launch(): - with (STATE / "stdout.log").open("wb") as output: - process = subprocess.Popen( - ["/usr/local/bin/thread-fuzz", "/usr/local/bin/reth-trie-dst", - "@" + str(STATE / "init.json")], stdout=output, stderr=subprocess.STDOUT) - tmp = STATE / "warm.pid.tmp" - tmp.write_text(str(process.pid)) - tmp.replace(STATE / "warm.pid") - code = process.wait() - tmp = STATE / "warm.exit.tmp" - tmp.write_text(str(code)) - tmp.replace(STATE / "warm.exit") - - -def warm_failure(message): - details = [message] - for name in ("launcher.pid", "warm.pid", "warm.exit", "launcher.log", "stdout.log"): - path = STATE / name - if path.exists(): - with path.open("rb") as source: - source.seek(max(0, path.stat().st_size - 12000)) - details.append(f"{name}: {source.read(12000).decode(errors='replace')}") - return "; ".join(details) - - -def warmup(request): - args = arguments(request["args"]) - if args.pop("cold_start"): - return {"ready": True, "detail": "cold_start: initialization remains in each execution"} - args.pop("crashes") - STATE.mkdir(parents=True, exist_ok=True) - initial = args | {"datadir": str(STATE / "data"), "init_socket": str(STATE / "init.sock")} - path = STATE / "init.json" - if path.exists(): - if json.loads(path.read_text()) != initial: - raise RuntimeError("warmup arguments changed") - else: - path.write_text(json.dumps(initial)) - (STATE / "warm.deadline").write_text(str(time.monotonic() + 120)) - with (STATE / "launcher.log").open("w") as output: - process = subprocess.Popen([sys.executable, __file__, "launch"], - stdout=output, stderr=subprocess.STDOUT, - start_new_session=True) - (STATE / "launcher.pid").write_text(str(process.pid)) - expired = time.monotonic() >= float((STATE / "warm.deadline").read_text()) - pid = STATE / "warm.pid" - if not alive(int((STATE / "launcher.pid").read_text())): - raise RuntimeError(warm_failure("warm launcher exited")) - if not pid.exists(): - if expired: - raise TimeoutError(warm_failure("warm initialization timed out")) - return {"ready": False, "detail": "launching worker"} - if not alive(int(pid.read_text())): - raise RuntimeError(warm_failure("warm worker exited before start")) - if not (STATE / "init.sock").exists(): - if expired: - raise TimeoutError(warm_failure("warm initialization timed out")) - return {"ready": False, "detail": "initializing runtime and empty storage"} - try: - barrier("ready", b"initialized") - except (OSError, RuntimeError) as error: - raise RuntimeError(warm_failure(f"warm readiness handshake failed: {error}")) from error - # The acknowledged worker is idle, so its output offset is stable until start. - prefix = (STATE / "stdout.log").read_bytes() - (STATE / "init.log").write_bytes(prefix) - (STATE / "warm-ready.json").write_text(json.dumps({ - "phase": "initialized/awaiting-input", "pid": int(pid.read_text()), - "stdout_offset": len(prefix)})) - return {"ready": True, "detail": "initialized/awaiting-input: live worker blocked on execution config"} - - -def read_output(source, process): - data = source.read(65536) - if data: - return data - if process.poll() is not None: - # Exit may race the first EOF read; drain after observing termination. - return source.read(65536) - return None - - -def worker(config): - process = None - killed = 0 - recovered = 0 - recovered_persisted = 0 - plan = list(config["kill_blocks"]) - require_persisted = (bool(plan) and - config["workload"]["blocks"] >= config["workload"]["persistence_threshold"] + 4) - persisted = False - # Rust may make two sequential timeout-bounded waits before its next event; - # recovery has no internal timeout and can use the campaign horizon. - idle_budget = 2 * config["workload"]["timeout_secs"] + config["run_secs"] - deadline = time.monotonic() + idle_budget - waiting = False - expected_blocks = config["workload"]["blocks"] - try: - workload = config["workload"] | {"datadir": str(STATE / "data"), - "input_file": str(STATE / "pending-input.json")} - workload_path = STATE / "workload.json" - workload_path.write_text(json.dumps(workload)) - warm = (STATE / "warm.pid").exists() - with (STATE / "run.log").open("wb") as output: - while True: - completed = False - offset = 0 - if warm: - process = WarmProcess() - ready = json.loads((STATE / "warm-ready.json").read_text()) - if ready["pid"] != process.pid: - raise RuntimeError("initialized worker PID changed") - offset = ready["stdout_offset"] - output.write((json.dumps({"warm": "reused", "pid": process.pid, - "stdout_offset": offset}) + "\n").encode()) - output.flush() - try: - barrier(str(workload_path), b"started", - timeout=max(0.001, deadline - time.monotonic()), process=process) - except (OSError, RuntimeError) as error: - raise RuntimeError(warm_failure(f"warm start handshake failed: {error}")) from error - warm = False - else: - with (STATE / "stdout.log").open("wb") as stdout: - process = subprocess.Popen( - ["/usr/local/bin/thread-fuzz", "/usr/local/bin/reth-trie-dst", - "@" + str(workload_path)], stdout=stdout, stderr=subprocess.STDOUT) - source = (STATE / "stdout.log").open("rb") - source.seek(offset) - pending = b"" - planned = False - while True: - remaining = deadline - time.monotonic() - if waiting: - remaining = 1 - elif remaining <= 0: - raise TimeoutError("workload recovery campaign timed out") - data = read_output(source, process) - if data == b"": - break - if data is None: - time.sleep(min(0.01, remaining)) - continue - output.write(data) - pending += data - while b"\n" in pending: - line, pending = pending.split(b"\n", 1) - try: - event = json.loads(line) - except (ValueError, UnicodeDecodeError): - continue # Rust errors/backtraces share stdout with JSON events. - if not isinstance(event, dict): - continue - deadline = time.monotonic() + idle_budget - if event.get("input") == "waiting": - waiting = True - assertion(True, json.dumps(event), "trie/input/checked", "Sometimes") - if event.get("input") == "restored": - expected_blocks = event["blocks"] - if event.get("input") == "injected": - waiting = False - # An injection replaces the remaining suffix, rather than appending it. - expected_blocks = event["block"] - 1 + event["blocks"] - if event.get("input") in ("injected", "restored"): - # A shorter suffix can make pending crash targets unreachable. - plan = [block for block in plan if block <= expected_blocks] - require_persisted = (bool(plan or require_persisted) and expected_blocks >= - config["workload"]["persistence_threshold"] + 4) - if event.get("completed"): - completed = True - assertion(event.get("blocks") == expected_blocks, - json.dumps(event), "trie/input/completed") - if event.get("input") == "injected": - assertion(True, json.dumps(event), "trie/input/injected", "Sometimes") - if event.get("persistence") == "complete": - assertion(True, json.dumps(event), "trie/persistence/committed", "Sometimes") - persisted |= event.get("db_tip", 0) > 0 - if event.get("recovery") == "complete": - recovered += 1 - recovered_persisted += int(killed > 0 and event.get("head", 0) > 0) - assertion(True, json.dumps(event), "trie/recovery/checked", "Sometimes") - position = event.get("block", event.get("head", 0)) - if (not planned and plan and position >= plan[0] and - ("operations" in event or - event.get("persistence") == "complete") and - (persisted or not require_persisted)): - process.send_signal(signal.SIGKILL) - target = plan.pop(0) - killed += 1 - planned = True - output.write((json.dumps({"nemesis": "kill", "target": target, - "block": position, "kills": killed}) + "\n").encode()) - output.flush() - code = process.wait(timeout=max(0.001, deadline - time.monotonic())) - source.close() - if planned: - if code != -signal.SIGKILL: - raise RuntimeError(f"planned SIGKILL did not kill worker: exit={code}") - continue - if code != 0: - raise RuntimeError(f"unexpected worker exit={code}") - if not completed: - raise RuntimeError("worker exited without completing its input") - if plan: - raise RuntimeError(f"nemesis did not fire at blocks {plan}") - break - ok = True - detail = f"completed after {killed} SIGKILL/restarts" - except Exception as error: - ok, detail = False, str(error) - finally: - if process is not None and process.poll() is None: - process.kill() - process.wait(timeout=5) - if not ok: - detail += "; " + (STATE / "run.log").read_text(errors="replace")[-12000:] - assertion(ok, detail) - if require_persisted: - assertion(recovered_persisted > 0, f"post-crash recovery from persisted head: {recovered_persisted}", - "trie/recovery/persisted") - (STATE / "done.json").write_text(json.dumps({"ok": ok, "kills": killed, - "armed": killed + len(plan), "recoveries": recovered, - "persisted_recoveries": recovered_persisted})) - - -def main(): - hook = sys.argv[1] - value = sys.argv[2] if len(sys.argv) > 2 else None - request = (json.load(sys.stdin) if value == "-" else - json.loads(Path(value[1:]).read_text()) if value and value.startswith("@") else - json.loads(value) if value else None) - if hook == "mutation-operators": - return native(hook, {}) - if hook in ("seed-inputs", "suffix-seed"): - args = arguments(request["args"]) - args.pop("crashes") - args.pop("cold_start") - if hook == "suffix-seed": - # Each block establishes its own accounts, so it also runs at a deep snapshot. - args.update(blocks=4, updates=min(4080, args["updates"])) - inputs = [json.dumps(native("generate-suffix" if hook == "suffix-seed" else "generate-input", args | {"seed": seed})) - for seed in range(1 if hook == "suffix-seed" else 8)] - return inputs[0] if hook == "suffix-seed" else inputs - if hook == "parse-args": - values = json.loads(Path(request["config_file"]).read_text()) if request.get("config_file") else {} - for pair in request.get("args", []): - key, value = pair.split("=", 1) - values[key] = json.loads(value) - return arguments(values) - if hook == "config": - if request.get("redraw") or request.get("drop"): - raise ValueError("this workload has no mid-run semantic decisions to redraw or drop") - decisions = request.get("decisions") - if decisions is not None and request.get("base_input") is not None: - raise ValueError("base_input cannot replace recorded decisions") - if decisions is None: - args = arguments(request["args"]) - crashes = args.pop("crashes") - cold_start = args.pop("cold_start") - supplied = request.get("base_input") - if supplied is not None and len(supplied) > 32 << 20: - raise ValueError("explicit input exceeds 32 MiB") - input_value = json.loads(supplied) if supplied is not None else None - if input_value is not None: - validation = native("validate-input", {"input": input_value}) - args.update(blocks=validation["blocks"], expected_roots=validation["expected_roots"]) - kill_blocks = sorted(random.Random(request["seed"]).sample( - range(1, args["blocks"] + 1), min(crashes, args["blocks"]))) - config = {"workload": args | {"seed": request["seed"], "input": input_value}, - "kill_blocks": kill_blocks, "cold_start": cold_start, - "run_secs": request["campaign"]["run_secs"]} - decisions = {"config": config, "seed": request["seed"], - "rng_seed": request["seed"], "preempt": request["preempt"]} - decisions = dict(decisions) - if request.get("rng_seed") is not None: - decisions["rng_seed"] = request["rng_seed"] - return {"config": decisions["config"], "decisions": decisions, - "run": {"seed": decisions["seed"], "rng_seed": decisions["rng_seed"], - "run_secs": decisions["config"]["run_secs"], - "preempt": decisions["preempt"], "swarm": {}}, - "finalized_by_start": decisions["config"]["run_secs"] == 1, - "required": ["trie/input/completed"], "replays_exactly": True, "action": None, - "notes": [], "host_decisions": []} - if hook == "warmup": - return warmup(request) - if hook == "start": - if not request.get("cold_start", False) and not (STATE / "warm.pid").exists(): - raise RuntimeError("warm worker was not initialized") - STATE.mkdir(parents=True, exist_ok=True) - if (STATE / "worker.pid").exists() or (STATE / "done.json").exists(): - raise RuntimeError("workload already started") - if request["run_secs"] == 1: - (STATE / "pending-input.finish").touch() - worker(request) - collect(Path("/bedrock/out/assertions.jsonl")) - else: - config_path = STATE / "config.json" - config_path.write_text(json.dumps(request)) - with (STATE / "controller.log").open("w") as output: - process = subprocess.Popen([sys.executable, __file__, "worker", "@" + str(config_path)], - stdout=output, stderr=subprocess.STDOUT, start_new_session=True) - (STATE / "worker.pid").write_text(str(process.pid)) - elif hook == "launch": - launch() - elif hook == "worker": - worker(request) - elif hook == "set-input": - if (STATE / "done.json").exists(): - raise RuntimeError("input injection reached an already completed workload") - pending = STATE / "pending-input.json" - tmp = pending.with_suffix(".tmp") - applied = STATE / "data/input-overrides.json" - previous = [json.loads(path.read_text())[key] for path, key in - ((pending, "id"), (applied, "applied")) if path.exists()] - tmp.write_text(json.dumps({"id": max(previous, default=0) + 1, "input": request})) - tmp.replace(pending) - elif hook == "merge-assertions": - collect(Path("/bedrock/out/assertions.jsonl")) - elif hook == "progress": - collect(Path("/bedrock/out/progress.jsonl"), progress=True) - elif hook == "finalize": - if STATE.exists(): - (STATE / "pending-input.finish").touch() - deadline = time.monotonic() + 5 - while STATE.exists() and not (STATE / "done.json").exists() and time.monotonic() < deadline: - time.sleep(0.01) - if not (STATE / "done.json").exists(): - pid = STATE / "worker.pid" - if pid.exists(): - try: - os.killpg(int(pid.read_text()), signal.SIGKILL) - except ProcessLookupError: - pass - warm_pid = STATE / "warm.pid" - if warm_pid.exists() and alive(int(warm_pid.read_text())): - try: - os.kill(int(warm_pid.read_text()), signal.SIGKILL) - except ProcessLookupError: - pass - log = STATE / "run.log" - assertion(False, "workload incomplete; " + (log.read_text()[-12000:] if log.exists() else "never started")) - collect(Path("/bedrock/out/assertions.jsonl")) - else: - raise ValueError(f"unknown hook: {hook}") - - -if __name__ == "__main__": - answer = main() - if answer is not None: - print(json.dumps(answer)) diff --git a/contrib/trie-dst/workload/prepare.sh b/contrib/trie-dst/workload/prepare.sh deleted file mode 100755 index 09a450ccfb4..00000000000 --- a/contrib/trie-dst/workload/prepare.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -root=$(cd "$(dirname "$0")/.." && pwd) -: "${RUN_DIR:?Bedrock must supply RUN_DIR}" -: "${BEDROCK_ROOT:?Bedrock must supply BEDROCK_ROOT}" -case "$*" in - --workload-config\ *) test "$#" = 2 ;; - *) echo 'expected --workload-config FILE' >&2; exit 1 ;; -esac -# Validate before building; parse-args is also used by the host driver. -python3 -c 'import json,sys; print(json.dumps({"config_file":sys.argv[1],"args":[]}))' "$2" | "$root/workload/controller.py" parse-args - >/dev/null -read -r -a docker_cmd <<< "${DOCKER:-docker}" -cache=${BEDROCK_DST_BUILD_CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/reth-trie-dst} -sccache=${BEDROCK_DST_SCCACHE_DIR:-$cache/sccache} -mkdir -p "$cache" "$sccache" "$RUN_DIR" -cache=$(cd "$cache" && pwd) -sccache=$(cd "$sccache" && pwd) -# Share the lock with Bedrock's cache pruning. -exec 9>"$cache/build.lock" -flock 9 -context=$(mktemp -d) -build_started=$SECONDS -trap 'printf "prepare_secs=%s\n" "$((SECONDS - build_started))" >> "$RUN_DIR/cache-info.env"; rm -rf "$context"' EXIT -printf 'cache_dir=%s\nsccache_dir=%s\n' "$cache" "$sccache" > "$RUN_DIR/cache-info.env" -printf '%s\n' bedrock/reth-trie-dst-builder:rust1.96-sccache0.14 bedrock/reth-trie-dst:latest > "$RUN_DIR/docker-images" -"${docker_cmd[@]}" build -f - -t bedrock/reth-trie-dst-builder:rust1.96-sccache0.14 "$context" <<'DOCKERFILE' -FROM rust:1.96-bookworm@sha256:19817ead3289c8c631c73df281e18b59b172f6a31f4f563290f69cddd06c30e9 -RUN apt-get update && apt-get install -y --no-install-recommends clang libclang-dev mold \ - && rm -rf /var/lib/apt/lists/* -RUN cargo install sccache --version 0.14.0 --locked -DOCKERFILE -# Stable paths let fresh CI checkouts reuse compiler results. -"${docker_cmd[@]}" run --rm -i --user "$(id -u):$(id -g)" \ - --mount "type=bind,src=$(cd "$root/../.." && pwd),dst=/workspace,readonly" \ - --mount "type=bind,src=$cache,dst=/cache" \ - --mount "type=bind,src=$sccache,dst=/cache/sccache" \ - --mount "type=bind,src=$context,dst=/out" \ - --mount "type=bind,src=$RUN_DIR,dst=/run-info" \ - --mount "type=bind,src=$BEDROCK_ROOT/guest,dst=/bedrock-guest,readonly" \ - --mount "type=bind,src=$BEDROCK_ROOT/scripts/dst/rust-coverage.sh,dst=/bedrock-rust-coverage.sh,readonly" \ - -e CARGO_BUILD_JOBS="${CARGO_BUILD_JOBS:-6}" \ - -e SCCACHE_CACHE_SIZE="${SCCACHE_CACHE_SIZE:-10G}" \ - -e DST_CMD="${DST_CMD:-campaign}" \ - -w /workspace bedrock/reth-trie-dst-builder:rust1.96-sccache0.14 bash -seu <<'BUILD' -export HOME=/cache CARGO_HOME=/cache/cargo CARGO_TARGET_DIR=/cache/target/reth-trie-dst -export SCCACHE_DIR=/cache/sccache RUSTC_WRAPPER=sccache CARGO_INCREMENTAL=0 -export CC='sccache gcc' CXX='sccache g++' RUSTFLAGS='-C link-arg=-fuse-ld=mold' -mkdir -p "$CARGO_HOME" "$CARGO_TARGET_DIR" "$SCCACHE_DIR" -touch "$CARGO_HOME" "$CARGO_TARGET_DIR" -sccache --zero-stats >/dev/null -trap 'sccache --show-stats > /run-info/sccache.txt; cat /run-info/sccache.txt' EXIT -# Keep the DST profile: it inherits profiling and enables debug assertions and -# overflow checks for the harness. Do not switch these builds to profiling. -if [ "${DST_CMD:-campaign}" = fuzz ]; then - export CARGO_PROFILE_DST_LTO=fat CARGO_PROFILE_DST_CODEGEN_UNITS=1 - source /bedrock-rust-coverage.sh - binary=$(bedrock_rust_coverage_build /bedrock-guest /out reth-trie-dst dst \ - --inline --locked -p reth-trie-dst --features inline-coverage) - bedrock_rust_coverage_check "$binary" --version -else - cargo build --locked -p reth-trie-dst --profile dst - binary="$CARGO_TARGET_DIR/dst/reth-trie-dst" -fi -printf 'cargo_secs=%s\n' "$SECONDS" >> /run-info/cache-info.env -# Keep symbols in the cached executable for debugging recorded failures. -objcopy --strip-debug "$binary" /run-info/reth-trie-dst -objcopy --strip-debug "$binary" /out/reth-trie-dst -sccache gcc -O2 -I /bedrock-guest -x c - -o /out/bedrock-ready <<'C' -#include "libvmcall.h" -int main(void) { - vmcall_ready(); - return 0; -} -C -BUILD -rm -f "$context/libfeedback.o" "$context/frontend.o" "$context/rt.a" -cp "$root/workload/controller.py" "$context/controller.py" -"${docker_cmd[@]}" build -f - -t bedrock/reth-trie-dst:latest "$context" <<'DOCKERFILE' -FROM debian:trixie-slim -RUN apt-get update && apt-get install -y --no-install-recommends python3 libstdc++6 && rm -rf /var/lib/apt/lists/* -COPY reth-trie-dst bedrock-ready controller.py /usr/local/bin/ -ENTRYPOINT ["/bin/sh", "-c", "/usr/local/bin/bedrock-ready && exec sleep infinity"] -DOCKERFILE -mkdir -p "$RUN_DIR" -cp "$root/workload/compose.yaml" "$RUN_DIR/compose.yaml" -cp "$root/workload/controller.py" "$RUN_DIR/controller.py" -"${docker_cmd[@]}" save bedrock/reth-trie-dst:latest > "$RUN_DIR/images.raw.tar" -python3 "$BEDROCK_ROOT/scripts/dst/oci_archive.py" normalize "$RUN_DIR/images.raw.tar" -o "$RUN_DIR/images.tar" -rm "$RUN_DIR/images.raw.tar" -printf '%s\n' "$RUN_DIR/images.tar" > "$RUN_DIR/images.path" diff --git a/crates/chainspec/src/lib.rs b/crates/chainspec/src/lib.rs index 902b6a92d19..e2b7f0ef658 100644 --- a/crates/chainspec/src/lib.rs +++ b/crates/chainspec/src/lib.rs @@ -38,6 +38,9 @@ pub use spec::{ #[cfg(test)] mod tests { use super::*; + use crate::{ChainSpec, EthChainSpec}; + use alloy_consensus::Header; + use alloy_eips::eip1559::INITIAL_BASE_FEE; use alloy_primitives::U256; use alloy_rlp::Encodable; use std::str::FromStr; @@ -140,10 +143,6 @@ mod tests { #[test] fn test_centralized_base_fee_calculation() { - use crate::{ChainSpec, EthChainSpec}; - use alloy_consensus::Header; - use alloy_eips::eip1559::INITIAL_BASE_FEE; - fn parent_header() -> Header { Header { gas_used: 15_000_000, diff --git a/crates/chainspec/src/spec.rs b/crates/chainspec/src/spec.rs index b18c53251b1..f4571135ed1 100644 --- a/crates/chainspec/src/spec.rs +++ b/crates/chainspec/src/spec.rs @@ -40,6 +40,7 @@ use reth_ethereum_forks::{ }; use reth_network_peers::{holesky_nodes, hoodi_nodes, mainnet_nodes, sepolia_nodes, NodeRecord}; use reth_primitives_traits::{sync::LazyLock, BlockHeader, SealedHeader}; +use NamedChain as C; /// Helper method building a [`Header`] given [`Genesis`] and [`ChainHardforks`]. pub fn make_genesis_header(genesis: &Genesis, hardforks: &ChainHardforks) -> Header { @@ -770,8 +771,6 @@ impl ChainSpec { /// Returns the known bootnode records for the given chain. pub fn bootnodes(&self) -> Option> { - use NamedChain as C; - match self.chain.try_into().ok()? { C::Mainnet => Some(mainnet_nodes()), C::Sepolia => Some(sepolia_nodes()), diff --git a/crates/cli/commands/src/db/mod.rs b/crates/cli/commands/src/db/mod.rs index cb3f08f7340..72427e2cc3f 100644 --- a/crates/cli/commands/src/db/mod.rs +++ b/crates/cli/commands/src/db/mod.rs @@ -181,9 +181,15 @@ impl> Command }); } Subcommands::RepairTrie(command) => { - let access_rights = - if command.dry_run { AccessRights::RO } else { AccessRights::RW }; - db_exec!(self.env, tool, N, access_rights, { + let mut env = self.env; + let access_rights = if command.dry_run { + // Separate worker transactions must open without concurrent commits. + env.db.exclusive = Some(true); + AccessRights::RO + } else { + AccessRights::RW + }; + db_exec!(env, tool, N, access_rights, { command.execute(&tool, ctx.task_executor, &data_dir)?; }); } diff --git a/crates/cli/commands/src/db/repair_trie.rs b/crates/cli/commands/src/db/repair_trie.rs index 602f8bbb4b4..4502cc18d6f 100644 --- a/crates/cli/commands/src/db/repair_trie.rs +++ b/crates/cli/commands/src/db/repair_trie.rs @@ -1,11 +1,11 @@ use alloy_consensus::BlockHeader as AlloyBlockHeader; -use clap::Parser; +use clap::{builder::RangedU64ValueParser, Parser}; use metrics::{self, Counter}; use reth_chainspec::EthChainSpec; use reth_cli_util::parse_socket_address; +use reth_db::DatabaseEnv; use reth_db_api::{ cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO}, - database::Database, transaction::{DbTx, DbTxMut}, }; use reth_db_common::DbTool; @@ -26,11 +26,11 @@ use reth_stages::StageId; use reth_storage_api::StorageSettingsCache; use reth_tasks::TaskExecutor; use reth_trie::{ - verify::{Output, Verifier}, + verify::{default_storage_workers, Output, DEFAULT_STORAGE_LOOKAHEAD}, Nibbles, }; use reth_trie_db::{ - DatabaseHashedCursorFactory, DatabaseStateRoot, DatabaseTrieCursorFactory, + DatabaseHashedCursorFactory, DatabaseStateRoot, DatabaseTrieCursorFactory, DatabaseVerifier, StorageTrieEntryLike, TrieTableAdapter, }; use std::{ @@ -44,10 +44,20 @@ const PROGRESS_PERIOD: Duration = Duration::from_secs(5); /// The arguments for the `reth db repair-trie` command #[derive(Parser, Debug)] pub struct Command { - /// Only show inconsistencies without making any repairs + /// Only show inconsistencies without making any repairs. + /// + /// Requires the node to be stopped so all workers read the same database snapshot. #[arg(long)] pub(crate) dry_run: bool, + /// Number of storage verification workers. Defaults to the number of available CPUs. + #[arg(long, default_value_t = default_storage_workers(), hide_default_value = true, value_parser = RangedU64ValueParser::::new().range(1..))] + pub(crate) storage_workers: usize, + + /// Maximum storage roots computed or in flight ahead of the account walk. + #[arg(long, default_value_t = DEFAULT_STORAGE_LOOKAHEAD, value_parser = RangedU64ValueParser::::new().range(1..))] + pub(crate) storage_lookahead: usize, + /// Enable Prometheus metrics. /// /// The metrics will be served at the given interface and port. @@ -57,7 +67,7 @@ pub struct Command { impl Command { /// Execute `db repair-trie` command - pub fn execute( + pub fn execute>( self, tool: &DbTool, task_executor: TaskExecutor, @@ -98,16 +108,20 @@ impl Command { }; if self.dry_run { - verify_only(tool)? + verify_only(tool, self.storage_workers, self.storage_lookahead)? } else { - verify_and_repair(tool)? + verify_and_repair(tool, self.storage_workers, self.storage_lookahead)? } Ok(()) } } -fn verify_only(tool: &DbTool) -> eyre::Result<()> { +fn verify_only>( + tool: &DbTool, + storage_workers: usize, + storage_lookahead: usize, +) -> eyre::Result<()> { // Log the database block tip from Finish stage checkpoint let finish_checkpoint = tool .provider_factory @@ -116,19 +130,21 @@ fn verify_only(tool: &DbTool) -> eyre::Result<()> { .unwrap_or_default(); info!("Database block tip: {}", finish_checkpoint.block_number); - // Get a database transaction directly from the database let db = tool.provider_factory.db_ref(); - let mut tx = db.tx()?; - tx.disable_long_read_transaction_safety(); - - reth_trie_db::with_adapter!(tool.provider_factory, |A| do_verify_only::<_, A>(&tx)) + reth_trie_db::with_adapter!(tool.provider_factory, |A| do_verify_only::( + db, + storage_workers, + storage_lookahead + )) } -fn do_verify_only(tx: &TX) -> eyre::Result<()> { - // Create the verifier - let hashed_cursor_factory = DatabaseHashedCursorFactory::new(tx); - let trie_cursor_factory = DatabaseTrieCursorFactory::<_, A>::new(tx); - let verifier = Verifier::new(&trie_cursor_factory, hashed_cursor_factory)?; +fn do_verify_only( + db: &DatabaseEnv, + storage_workers: usize, + storage_lookahead: usize, +) -> eyre::Result<()> { + let db = db.clone().with_disable_long_read_transaction_safety(); + let verifier = DatabaseVerifier::<_, A>::new(&db, storage_workers, storage_lookahead)?; let metrics = RepairTrieMetrics::new(); @@ -207,7 +223,11 @@ fn verify_checkpoints(provider: impl StageCheckpointReader) -> eyre::Result<()> Ok(()) } -fn verify_and_repair(tool: &DbTool) -> eyre::Result<()> { +fn verify_and_repair>( + tool: &DbTool, + storage_workers: usize, + storage_lookahead: usize, +) -> eyre::Result<()> { // Get a read-write database provider let mut provider_rw = tool.provider_factory.provider_rw()?; @@ -219,7 +239,13 @@ fn verify_and_repair(tool: &DbTool) -> eyre::Result<()> verify_checkpoints(provider_rw.as_ref())?; let inconsistent_nodes = reth_trie_db::with_adapter!(tool.provider_factory, |A| { - do_verify_and_repair::<_, A>(&mut provider_rw, finish_checkpoint.block_number)? + do_verify_and_repair::<_, A>( + &mut provider_rw, + tool.provider_factory.db_ref(), + finish_checkpoint.block_number, + storage_workers, + storage_lookahead, + )? }); if inconsistent_nodes == 0 { @@ -232,27 +258,22 @@ fn verify_and_repair(tool: &DbTool) -> eyre::Result<()> Ok(()) } -fn do_verify_and_repair( +fn do_verify_and_repair, A: TrieTableAdapter>( provider_rw: &mut reth_provider::DatabaseProviderRW, + db: &N::DB, block_number: u64, -) -> eyre::Result -where - ::TXMut: DbTxMut + DbTx, -{ + storage_workers: usize, + storage_lookahead: usize, +) -> eyre::Result { // Create cursors for making modifications with let tx = provider_rw.tx_mut(); tx.disable_long_read_transaction_safety(); let mut account_trie_cursor = tx.cursor_write::()?; let mut storage_trie_cursor = tx.cursor_dup_write::()?; - // Create the cursor factories. These cannot accept the `&mut` tx above because they - // require it to be AsRef. - let tx = provider_rw.tx_ref(); - let hashed_cursor_factory = DatabaseHashedCursorFactory::new(tx); - let trie_cursor_factory = DatabaseTrieCursorFactory::<_, A>::new(tx); - - // Create the verifier - let verifier = Verifier::new(&trie_cursor_factory, hashed_cursor_factory)?; + // Open every read snapshot before any repairs are applied. + let db = db.clone().with_disable_long_read_transaction_safety(); + let verifier = DatabaseVerifier::<_, A>::new(&db, storage_workers, storage_lookahead)?; let metrics = RepairTrieMetrics::new(); diff --git a/crates/cli/commands/src/download/extract.rs b/crates/cli/commands/src/download/extract.rs index fb478b80c2e..73fa7e8b69c 100644 --- a/crates/cli/commands/src/download/extract.rs +++ b/crates/cli/commands/src/download/extract.rs @@ -122,10 +122,8 @@ fn unpack_archive( static_files_dir: Option<&Path>, mut progress: Option<&mut ArchiveExtractionProgress>, ) -> Result<()> { - if static_files_dir.is_none() && progress.is_none() { - archive.unpack(target_dir)?; - return Ok(()) - } + // Every entry goes through `unpack_entry` so link entries are rejected; `Archive::unpack` + // would create them. let entries = archive.entries().wrap_err_with(|| { format!("failed to read archive entries for `{}`", target_dir.display()) })?; @@ -150,6 +148,27 @@ pub(crate) fn static_file_relative_path(path: &Path) -> Option<&Path> { path.strip_prefix(".").unwrap_or(path).strip_prefix("static_files").ok() } +/// Fails if any existing component of `relative_path` below `root` is a symlink. +/// +/// Components that do not exist yet are ignored. `root` itself is trusted and not checked. This +/// is a point-in-time check: it does not stop a link from being created right after it returns. +pub(crate) fn ensure_no_symlinks(root: &Path, relative_path: &Path) -> Result<()> { + let mut current = root.to_path_buf(); + for part in relative_path.components() { + current.push(part); + match std::fs::symlink_metadata(¤t) { + Ok(metadata) => eyre::ensure!( + !metadata.file_type().is_symlink(), + "Archive path `{}` contains a symlink", + current.display() + ), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => return Err(err.into()), + } + } + Ok(()) +} + /// Extracts static files beneath their configured root without allowing archive paths or links /// to escape it. fn unpack_static_file( @@ -173,18 +192,7 @@ fn unpack_static_file( let root = static_files_dir.to_path_buf(); let dest = root.join(relative_path); // Check each existing ancestor before creating directories or replacing an output. - let mut current = root.clone(); - for part in relative_path.components() { - current.push(part); - match std::fs::symlink_metadata(¤t) { - Ok(metadata) => eyre::ensure!( - !metadata.file_type().is_symlink(), - "Static file archive path contains a symlink" - ), - Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} - Err(err) => return Err(err.into()), - } - } + ensure_no_symlinks(&root, relative_path)?; if relative_path.as_os_str().is_empty() { return Ok(()) } @@ -200,11 +208,13 @@ fn unpack_entry( target_dir: &Path, static_files_dir: Option<&Path>, ) -> Result<()> { - // A non-static entry could otherwise plant a symlink at a nested custom root before - // extraction or retry cleanup uses it. + // Links planted by an archive could redirect later extraction, or retry cleanup running + // concurrently for another archive, outside the datadir. Snapshots never contain links. + let entry_type = entry.header().entry_type(); eyre::ensure!( - static_files_dir.is_none() || !entry.header().entry_type().is_symlink(), - "Archive symlinks are unsupported with a custom static files directory" + !entry_type.is_symlink() && !entry_type.is_hard_link(), + "Archive links are unsupported: `{}`", + entry.path()?.display() ); let path = entry.path()?.into_owned(); if let Some(static_files_dir) = static_files_dir && @@ -649,7 +659,7 @@ mod tests { let err = unpack_archive(Archive::new(tar.as_slice()), target.path(), Some(&custom), None) .unwrap_err(); - assert!(format!("{err:#}").contains("Archive symlinks are unsupported")); + assert!(format!("{err:#}").contains("Archive links are unsupported")); let outputs = [super::super::manifest::OutputFileChecksum { path: "static_files/headers".into(), size: 4, @@ -663,6 +673,49 @@ mod tests { } } + #[test] + fn link_entries_are_rejected_without_custom_static_dir() { + for entry_type in [tar::EntryType::Symlink, tar::EntryType::Link] { + for with_progress in [false, true] { + let root = tempfile::tempdir().unwrap(); + let target = root.path().join("target"); + let outside = root.path().join("outside"); + fs::create_dir_all(&target).unwrap(); + fs::create_dir_all(&outside).unwrap(); + fs::write(outside.join("file"), b"keep").unwrap(); + + let mut archive = tar::Builder::new(Vec::new()); + let mut link = tar::Header::new_gnu(); + link.set_entry_type(entry_type); + link.set_size(0); + link.set_mode(0o777); + // A symlink to the outside directory would redirect the entry below, a hard link + // would alias the outside file. + let link_target = + if entry_type.is_symlink() { outside.clone() } else { outside.join("file") }; + archive.append_link(&mut link, "db", link_target).unwrap(); + let mut file = tar::Header::new_gnu(); + file.set_size(4); + file.set_mode(0o644); + file.set_cksum(); + archive.append_data(&mut file, "db/file", b"data".as_slice()).unwrap(); + let tar = archive.into_inner().unwrap(); + + let mut progress = ArchiveExtractionProgress::new(None); + let progress = with_progress.then_some(&mut progress); + let err = unpack_archive(Archive::new(tar.as_slice()), &target, None, progress) + .unwrap_err(); + assert!(format!("{err:#}").contains("Archive links are unsupported")); + + assert!( + !target.join("db").exists() && target.join("db").symlink_metadata().is_err() + ); + assert_eq!(fs::read(outside.join("file")).unwrap(), b"keep"); + assert_eq!(fs::read_dir(&outside).unwrap().count(), 1); + } + } + } + #[test] fn remap_static_files_in_both_compression_formats() { let mut archive = tar::Builder::new(Vec::new()); diff --git a/crates/cli/commands/src/download/fetch.rs b/crates/cli/commands/src/download/fetch.rs index c278cf85a28..f3d5c5f156b 100644 --- a/crates/cli/commands/src/download/fetch.rs +++ b/crates/cli/commands/src/download/fetch.rs @@ -15,6 +15,7 @@ use std::{ collections::VecDeque, fs::OpenOptions, io::{self, BufWriter, Read, Write}, + os::unix::fs::FileExt, path::{Path, PathBuf}, sync::{ atomic::{AtomicBool, AtomicU64, Ordering}, @@ -770,8 +771,6 @@ impl SegmentedDownload { piece_progress_bytes: &AtomicU64, cancel_token: &CancellationToken, ) -> std::result::Result<(), PieceAttemptFailure> { - use std::os::unix::fs::FileExt; - let expected_len = piece.end - piece.start + 1; let response = match client diff --git a/crates/cli/commands/src/download/manifest.rs b/crates/cli/commands/src/download/manifest.rs index 72c63405093..865fc16ccfe 100644 --- a/crates/cli/commands/src/download/manifest.rs +++ b/crates/cli/commands/src/download/manifest.rs @@ -1,5 +1,5 @@ use blake3::Hasher; -use eyre::Result; +use eyre::{Result, WrapErr}; use rayon::prelude::*; use reqwest::Client; use reth_db::static_file::iter_static_files; @@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize}; use std::{ collections::BTreeMap, io::Read, - path::{Path, PathBuf}, + path::{Component, Path, PathBuf}, }; use tracing::info; use url::Url; @@ -150,6 +150,24 @@ pub struct OutputFileChecksum { pub blake3: String, } +impl OutputFileChecksum { + /// Returns the path as a relative path that stays below the directory it is joined to. + /// + /// Only normal components are accepted, plus the `./` prefix tar adds to entry paths. + /// Absolute paths and `..` are rejected because the path is joined onto the datadir for + /// verification and for removal before retries. + pub(crate) fn relative_path(&self) -> Result<&Path> { + let path = Path::new(&self.path); + eyre::ensure!( + path.components().all(|part| matches!(part, Component::Normal(_) | Component::CurDir)) && + path.components().any(|part| matches!(part, Component::Normal(_))), + "Invalid manifest output file path `{}`: must be a relative path without `..`", + self.path + ); + Ok(path) + } +} + /// A concrete snapshot archive with its download and verification metadata. #[derive(Debug, Clone, PartialEq, Eq)] pub struct SnapshotArchive { @@ -389,6 +407,25 @@ impl SnapshotManifest { } } + /// Fails if any output file path in the manifest is not a plain relative path. + /// + /// Output paths come from an untrusted manifest, so this is checked before any archive is + /// planned. + pub fn validate_output_paths(&self) -> Result<()> { + for (key, component) in &self.components { + let outputs: Box> = match component { + ComponentManifest::Single(single) => Box::new(single.output_files.iter()), + ComponentManifest::Chunked(chunked) => { + Box::new(chunked.chunk_output_files.iter().flatten()) + } + }; + for output in outputs { + output.relative_path().wrap_err_with(|| format!("In component `{key}`"))?; + } + } + Ok(()) + } + /// Returns the exact download size for a component given a distance ending at [`Self::block`]. /// /// For single archives, returns the full size. For chunked archives, sums the @@ -1320,6 +1357,46 @@ mod tests { assert_eq!(m.size_for_distance(SnapshotComponentType::Receipts, None), 0); } + #[test] + fn manifest_output_paths_must_stay_relative() { + fn output(path: &str) -> OutputFileChecksum { + OutputFileChecksum { path: path.into(), size: 0, blake3: String::new() } + } + + let mut manifest = test_manifest(); + let mut with_outputs = |single: &[&str], chunks: &[&str]| { + manifest.components.insert( + "state".into(), + ComponentManifest::Single(SingleArchive { + file: "state.tar.zst".into(), + size: 1, + decompressed_size: 0, + blake3: None, + output_files: single.iter().map(|path| output(path)).collect(), + }), + ); + manifest.components.insert( + "transactions".into(), + ComponentManifest::Chunked(ChunkedArchive { + blocks_per_file: 500_000, + total_blocks: 500_000, + chunk_sizes: vec![1], + chunk_decompressed_sizes: vec![], + chunk_files: vec![], + chunk_output_files: vec![chunks.iter().map(|path| output(path)).collect()], + }), + ); + manifest.validate_output_paths() + }; + + with_outputs(&["db/mdbx.dat", "./static_files/headers"], &["static_files/a/b"]).unwrap(); + for bad in ["/etc/passwd", "../outside", "db/../../outside", "db/..", ".", "", "./"] { + let err = with_outputs(&[bad], &[]).unwrap_err(); + assert!(format!("{err:#}").contains("Invalid manifest output file path"), "{bad}"); + assert!(with_outputs(&[], &[bad]).is_err(), "{bad}"); + } + } + #[test] fn output_size_for_distance_uses_manifest_or_output_files() { let m = test_manifest(); diff --git a/crates/cli/commands/src/download/prune.rs b/crates/cli/commands/src/download/prune.rs index 354f42b55bc..e82f1476d66 100644 --- a/crates/cli/commands/src/download/prune.rs +++ b/crates/cli/commands/src/download/prune.rs @@ -44,8 +44,8 @@ pub(crate) fn prune_unlisted_outputs( let listed = planned .iter() .flat_map(|planned| &planned.archive.output_files) - .map(|output| verifier.output_path(&output.path)) - .collect::>(); + .map(|output| verifier.output_path(output)) + .collect::>>()?; let mut removed = 0; for entry in MANAGED_DATADIR_PATHS { diff --git a/crates/cli/commands/src/download/source.rs b/crates/cli/commands/src/download/source.rs index c8ef9433e11..d462404738f 100644 --- a/crates/cli/commands/src/download/source.rs +++ b/crates/cli/commands/src/download/source.rs @@ -4,6 +4,7 @@ use super::{ use eyre::{Result, WrapErr}; use reqwest::Client; use reth_fs_util as fs; +use serde::Deserialize; use std::path::{Path, PathBuf}; use tracing::info; use url::Url; @@ -71,7 +72,6 @@ fn deserialize_string_or_u64<'de, D>(deserializer: D) -> std::result::Result, { - use serde::Deserialize; let value = serde_json::Value::deserialize(deserializer)?; match &value { serde_json::Value::Number(n) => { @@ -135,8 +135,15 @@ pub(crate) fn print_snapshot_listing(entries: &[SnapshotApiEntry], chain_id: u64 ); } -/// Loads a manifest from an HTTP(S) URL, `file://` URL, or local path. +/// Loads a manifest from an HTTP(S) URL, `file://` URL, or local path, and checks that its +/// output file paths stay inside the datadir. pub(crate) async fn fetch_manifest_from_source(source: &str) -> Result { + let manifest = read_manifest_source(source).await?; + manifest.validate_output_paths()?; + Ok(manifest) +} + +async fn read_manifest_source(source: &str) -> Result { if let Ok(parsed) = Url::parse(source) { return match parsed.scheme() { "http" | "https" => { diff --git a/crates/cli/commands/src/download/verify.rs b/crates/cli/commands/src/download/verify.rs index 897b6c415f0..2a0772d22ac 100644 --- a/crates/cli/commands/src/download/verify.rs +++ b/crates/cli/commands/src/download/verify.rs @@ -1,4 +1,7 @@ -use super::{manifest::OutputFileChecksum, progress::ArchiveVerificationProgress}; +use super::{ + extract::ensure_no_symlinks, manifest::OutputFileChecksum, + progress::ArchiveVerificationProgress, +}; use blake3::Hasher; use eyre::Result; use rayon::prelude::*; @@ -6,8 +9,9 @@ use reth_fs_util::{self as fs, FsPathError}; use std::{ io::{ErrorKind, Read}, mem, - path::Path, + path::{Path, PathBuf}, }; +use tracing::warn; /// Read size per hash update. Large enough for `update_rayon` to split across threads. const HASH_BUFFER_SIZE: usize = 4 * 1024 * 1024; @@ -48,7 +52,7 @@ impl<'a> OutputVerifier<'a> { let mut paths = Vec::with_capacity(output_files.len()); for expected in output_files { - let output_path = self.output_path(&expected.path); + let output_path = self.output_path(expected)?; match fs::metadata(&output_path) { Ok(meta) if meta.len() == expected.size => paths.push(output_path), Ok(_) => return Ok(false), @@ -73,20 +77,47 @@ impl<'a> OutputVerifier<'a> { } /// Removes any declared output files so a fresh archive attempt can restart cleanly. + /// + /// Files are only removed if their path stays below the output root and no existing parent + /// directory is a symlink. Extraction rejects link entries, so this guards against links + /// already present in the datadir; the check is not atomic with the removal. pub(crate) fn cleanup(&self, output_files: &[OutputFileChecksum]) { for output in output_files { - let _ = fs::remove_file(self.output_path(&output.path)); + match self.removal_path(output) { + Ok(path) => { + let _ = fs::remove_file(path); + } + Err(error) => { + warn!(target: "reth::cli", path = %output.path, %error, "Skipping output cleanup"); + } + } } } + /// Returns the path to remove for `output`, or an error if removing it could touch anything + /// outside the output root. + fn removal_path(&self, output: &OutputFileChecksum) -> Result { + let (root, relative) = self.resolve(output)?; + if let Some(parent) = relative.parent() { + ensure_no_symlinks(root, parent)?; + } + Ok(root.join(relative)) + } + /// Resolves archive paths consistently for verification and retry cleanup. - pub(crate) fn output_path(&self, path: &str) -> std::path::PathBuf { + pub(crate) fn output_path(&self, output: &OutputFileChecksum) -> Result { + self.resolve(output).map(|(root, relative)| root.join(relative)) + } + + /// Returns the output root and the validated path of `output` relative to it. + fn resolve<'p>(&'p self, output: &'p OutputFileChecksum) -> Result<(&'p Path, &'p Path)> { + let path = output.relative_path()?; if let Some(static_files_dir) = self.static_files_dir && - let Some(relative_path) = super::extract::static_file_relative_path(Path::new(path)) + let Some(relative_path) = super::extract::static_file_relative_path(path) { - return static_files_dir.join(relative_path) + return Ok((static_files_dir, relative_path)) } - self.target_dir.join(path) + Ok((self.target_dir, path)) } /// Computes the hex-encoded BLAKE3 checksum for one plain output file. @@ -217,4 +248,65 @@ mod tests { assert!(!custom_file.exists()); assert_eq!(fs::read(default_file).unwrap(), b"headers"); } + + #[test] + fn cleanup_ignores_output_paths_outside_target() { + let root = tempfile::tempdir().unwrap(); + let datadir = root.path().join("datadir"); + fs::create_dir_all(&datadir).unwrap(); + let outside = root.path().join("outside"); + fs::write(&outside, b"keep").unwrap(); + let inside = datadir.join("inside"); + fs::write(&inside, b"remove").unwrap(); + + let outputs = [ + checksum(outside.to_str().unwrap(), b"keep"), + checksum("../outside", b"keep"), + checksum("db/../../outside", b"keep"), + checksum("inside", b"remove"), + ]; + let verifier = OutputVerifier::new(&datadir, None); + assert!(verifier.verify(&outputs).is_err(), "invalid paths are never verified"); + verifier.cleanup(&outputs); + + assert_eq!(fs::read(&outside).unwrap(), b"keep"); + assert!(!inside.exists()); + } + + #[cfg(unix)] + #[test] + fn cleanup_does_not_follow_symlinked_parents() { + let root = tempfile::tempdir().unwrap(); + let datadir = root.path().join("datadir"); + let static_dir = root.path().join("static"); + let outside = root.path().join("outside"); + fs::create_dir_all(&datadir).unwrap(); + fs::create_dir_all(&static_dir).unwrap(); + fs::create_dir_all(outside.join("nested")).unwrap(); + for path in ["mdbx.dat", "headers", "nested/headers"] { + fs::write(outside.join(path), b"keep").unwrap(); + } + std::os::unix::fs::symlink(&outside, datadir.join("db")).unwrap(); + std::os::unix::fs::symlink(&outside, static_dir.join("link")).unwrap(); + std::os::unix::fs::symlink(&outside, static_dir.join("nested")).unwrap(); + + let outputs = [ + checksum("db/mdbx.dat", b"keep"), + checksum("static_files/link/headers", b"keep"), + checksum("static_files/nested/nested/headers", b"keep"), + ]; + for static_files_dir in [None, Some(static_dir.as_path())] { + OutputVerifier::new(&datadir, static_files_dir).cleanup(&outputs); + } + + for path in ["mdbx.dat", "headers", "nested/headers"] { + assert_eq!(fs::read(outside.join(path)).unwrap(), b"keep"); + } + + // A symlink as the output itself is unlinked without touching its target. + std::os::unix::fs::symlink(outside.join("mdbx.dat"), datadir.join("link")).unwrap(); + OutputVerifier::new(&datadir, None).cleanup(&[checksum("link", b"keep")]); + assert!(datadir.join("link").symlink_metadata().is_err()); + assert_eq!(fs::read(outside.join("mdbx.dat")).unwrap(), b"keep"); + } } diff --git a/crates/e2e-test-utils/src/trie.rs b/crates/e2e-test-utils/src/trie.rs index 8a407fba6d8..79027b32fdd 100644 --- a/crates/e2e-test-utils/src/trie.rs +++ b/crates/e2e-test-utils/src/trie.rs @@ -1,5 +1,6 @@ //! Helpers for verifying the persisted state and trie representation of test nodes. +use crate::TmpDB; use alloy_consensus::BlockHeader; use eyre::{ensure, eyre, Result}; use reth_provider::{ @@ -7,15 +8,15 @@ use reth_provider::{ }; use reth_trie::{ prefix_set::{PrefixSetMut, TriePrefixSets}, - verify::{Output, Verifier}, + verify::{default_storage_workers, Output, DEFAULT_STORAGE_LOOKAHEAD}, StateRoot, }; -use reth_trie_db::{DatabaseHashedCursorFactory, DatabaseTrieCursorFactory}; +use reth_trie_db::{DatabaseHashedCursorFactory, DatabaseTrieCursorFactory, DatabaseVerifier}; /// Asserts that the persisted state and trie representation is internally consistent and matches /// the state root of the latest persisted block. /// -/// This performs two checks against a single database snapshot: +/// This performs two checks against persisted database state: /// - the state root recomputed from the persisted hashed state matches the state root of the latest /// persisted header, and /// - the persisted trie nodes match a recomputation from the hashed state (the library equivalent @@ -28,9 +29,12 @@ use reth_trie_db::{DatabaseHashedCursorFactory, DatabaseTrieCursorFactory}; pub fn assert_trie_consistency(factory: &F) -> Result<()> where F: DatabaseProviderFactory< - Provider: DBProvider + StorageSettingsCache + BlockNumReader + HeaderProvider, - >, + DB = TmpDB, + Provider: DBProvider + StorageSettingsCache + BlockNumReader + HeaderProvider, + > + AsRef, { + // Prevent commits while opening the independent snapshots. + let snapshot_guard = factory.database_provider_rw()?; let provider = factory.database_provider_ro()?; let tip = provider.best_block_number()?; let header = provider @@ -39,8 +43,16 @@ where let tx = provider.tx_ref(); reth_trie_db::with_adapter!(provider, |A| { + let db = factory.as_ref().db().clone().with_disable_long_read_transaction_safety(); + let verifier = DatabaseVerifier::<_, A>::new( + &db, + default_storage_workers(), + DEFAULT_STORAGE_LOOKAHEAD, + )?; + drop(snapshot_guard); + // Recompute all account leaves from the persisted hashed state. Storage roots are taken - // from the persisted storage tries, which the verifier below checks against the hashed + // from the persisted storage tries, which the verifier checks against the hashed // storage tables. let recomputed = StateRoot::new( DatabaseTrieCursorFactory::<_, A>::new(tx), @@ -57,8 +69,6 @@ where header.state_root() ); - let trie_cursor_factory = DatabaseTrieCursorFactory::<_, A>::new(tx); - let verifier = Verifier::new(&trie_cursor_factory, DatabaseHashedCursorFactory::new(tx))?; let mut inconsistencies = Vec::new(); for output in verifier { match output? { diff --git a/crates/engine/invalid-block-hooks/src/witness.rs b/crates/engine/invalid-block-hooks/src/witness.rs index e6dae4bccc1..d1034958ca4 100644 --- a/crates/engine/invalid-block-hooks/src/witness.rs +++ b/crates/engine/invalid-block-hooks/src/witness.rs @@ -357,6 +357,7 @@ mod tests { use tempfile::TempDir; use reth_testing_utils::generators::{self, random_block, random_eoa_accounts, BlockParams}; + use std::collections::HashSet; /// Creates a test `BundleState` with realistic accounts, contracts, and reverts fn create_bundle_state() -> BundleState { @@ -637,7 +638,6 @@ mod tests { fn create_test_trie_updates() -> TrieUpdates { use alloy_primitives::map::HashMap; use reth_trie::{updates::TrieUpdates, BranchNodeCompact, Nibbles}; - use std::collections::HashSet; let mut account_nodes = HashMap::default(); let nibbles = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x3]); diff --git a/crates/engine/tree/src/persistence.rs b/crates/engine/tree/src/persistence.rs index f19afa6b48b..82e5ef0d10a 100644 --- a/crates/engine/tree/src/persistence.rs +++ b/crates/engine/tree/src/persistence.rs @@ -415,14 +415,15 @@ mod tests { use alloy_eips::NumHash; use alloy_primitives::{Address, BlockHash, BlockNumber, Bytes, B256, U256}; use reth_chain_state::{test_utils::TestBlockBuilder, ExecutedBlock}; + use reth_db::{models::ShardedKey, tables, BlockNumberList}; use reth_db_common::init::init_genesis; use reth_exex_types::FinishedExExHeight; use reth_provider::{ providers::{BlockchainProvider, ProviderFactoryBuilder, ReadOnlyConfig}, test_utils::{create_test_provider_factory, MockNodeTypes}, AccountReader, BalConfig, BalStore, BalStoreHandle, ChainSpecProvider, HeaderProvider, - InMemoryBalStore, ProviderError, ProviderResult, RawBal, StateProviderFactory, - StorageSettingsCache, + InMemoryBalStore, ProviderError, ProviderResult, RawBal, RocksDBBalStore, + RocksDBProviderFactory, StateProviderFactory, StorageSettingsCache, }; use reth_prune::Pruner; use reth_prune_types::PruneMode; @@ -564,8 +565,6 @@ mod tests { #[test] fn test_save_blocks_flushes_bal_store() { - use reth_provider::{RocksDBBalStore, RocksDBProviderFactory}; - reth_tracing::init_test_tracing(); let provider = create_test_provider_factory(); let bal_store = BalStoreHandle::new(RocksDBBalStore::new(provider.rocksdb_provider())); @@ -642,9 +641,6 @@ mod tests { /// first, then runs the pruner against committed state in a separate provider. #[test] fn test_save_blocks_then_prune_preserves_new_history() { - use reth_db::{models::ShardedKey, tables, BlockNumberList}; - use reth_provider::RocksDBProviderFactory; - reth_tracing::init_test_tracing(); let provider_factory = create_test_provider_factory(); diff --git a/crates/engine/tree/src/tree/payload_processor/bal/execute.rs b/crates/engine/tree/src/tree/payload_processor/bal/execute.rs index 4b3cbe6fcbb..0c1572b291a 100644 --- a/crates/engine/tree/src/tree/payload_processor/bal/execute.rs +++ b/crates/engine/tree/src/tree/payload_processor/bal/execute.rs @@ -230,7 +230,9 @@ mod tests { use super::*; use alloy_consensus::{BlockHeader, Header, TxLegacy}; - use alloy_eip7928::{bal::Bal as AlloyBal, BlockAccessList}; + use alloy_eip7928::{ + bal::Bal as AlloyBal, AccountChanges, BlockAccessIndex, BlockAccessList, CodeChange, + }; use alloy_eips::{ eip2935::{HISTORY_STORAGE_ADDRESS, HISTORY_STORAGE_CODE}, eip4788::{BEACON_ROOTS_ADDRESS, BEACON_ROOTS_CODE}, @@ -239,6 +241,7 @@ mod tests { use alloy_primitives::{Bytes, B256, U256}; use evm2::evm::{CacheDB, EmptyDB}; use reth_chainspec::{ChainSpecBuilder, MAINNET}; + use reth_engine_primitives::{InsertBlockErrorKind, InsertBlockValidationError}; use reth_ethereum_primitives::{Block, BlockBody, Receipt, Transaction, TransactionSigned}; use reth_evm::{ cached::{AccountInfo, Bytecode}, @@ -249,7 +252,10 @@ mod tests { crypto::secp256k1::public_key_to_address, Block as _, Recovered, SealedBlock, }; use reth_testing_utils::generators::{generate_key, rng, sign_tx_with_key_pair}; - use std::convert::Infallible; + use std::{ + convert::Infallible, + sync::atomic::{AtomicUsize, Ordering}, + }; fn test_evm_config() -> EthEvmConfig { EthEvmConfig::new(Arc::new(ChainSpecBuilder::mainnet().amsterdam_activated().build())) @@ -498,8 +504,6 @@ mod tests { #[test] fn invalid_bal_bytecode_is_validation_error() { - use alloy_eip7928::{AccountChanges, BlockAccessIndex, CodeChange}; - use reth_engine_primitives::{InsertBlockErrorKind, InsertBlockValidationError}; let bal = vec![AccountChanges { address: Address::ZERO, code_changes: vec![CodeChange::new( @@ -730,7 +734,6 @@ mod tests { // Build the BAL an empty block actually produces, then append a phantom address // that execution never touches. The rebuilt BAL omits it, and the outer consensus // validator is responsible for comparing that rebuilt hash to the header commitment. - use alloy_eip7928::AccountChanges; let evm_config = test_evm_config(); @@ -1056,8 +1059,6 @@ mod tests { #[test] fn worker_reuses_provider_after_failure_and_finishes_earlier_transactions() { - use std::sync::atomic::{AtomicUsize, Ordering}; - let evm_config = test_evm_config(); let (pre_block_db, reference_bal, tx1, tx2) = two_transfers_with_reference_bal(&evm_config, 100_000, true); diff --git a/crates/engine/tree/src/tree/payload_processor/mod.rs b/crates/engine/tree/src/tree/payload_processor/mod.rs index 8b223270e92..44c56436148 100644 --- a/crates/engine/tree/src/tree/payload_processor/mod.rs +++ b/crates/engine/tree/src/tree/payload_processor/mod.rs @@ -708,15 +708,13 @@ impl CacheTaskHandle { &mut self, execution_outcome: Option>>, ) -> Option> { - if let Some(tx) = self.to_prewarm_task.take() { + self.to_prewarm_task.take().map(|tx| { let (valid_block_tx, valid_block_rx) = mpsc::channel(); let event = PrewarmTaskEvent::Terminate { execution_outcome, valid_block_rx }; let _ = tx.send(event); - Some(valid_block_tx) - } else { - None - } + valid_block_tx + }) } } diff --git a/crates/engine/tree/src/tree/payload_processor/prewarm.rs b/crates/engine/tree/src/tree/payload_processor/prewarm.rs index 3221a922bed..14c554d8d72 100644 --- a/crates/engine/tree/src/tree/payload_processor/prewarm.rs +++ b/crates/engine/tree/src/tree/payload_processor/prewarm.rs @@ -1037,6 +1037,7 @@ pub struct PrewarmMetrics { mod tests { use super::*; + use crate::tree::payload_processor::{CacheTaskHandle, PayloadHandle}; use alloy_consensus::{transaction::Recovered, TxLegacy}; use alloy_eips::eip7702::constants::EIP7702_CLEARED_DELEGATION; use alloy_primitives::{Address, Signature, TxKind, B256, U256}; @@ -1049,6 +1050,7 @@ mod tests { use reth_stages_api::{StageCheckpoint, StageId}; use reth_storage_overlay::OverlayManager; use reth_tasks::WorkerPool; + use reth_trie_common::bundle_state::{AccountStatus, BundleAccount, BundleState}; use std::time::Duration; #[test] @@ -1383,8 +1385,6 @@ mod tests { #[test] fn save_cache_freeing_waits_for_validator_cache_references() { - use crate::tree::payload_processor::{CacheTaskHandle, PayloadHandle}; - let runtime = Runtime::test(); let execution_cache = PayloadExecutionCache::default(); let saved = SavedCache::new(B256::repeat_byte(1), crate::tree::ExecutionCache::new(1_000)); @@ -1434,8 +1434,6 @@ mod tests { } fn assert_save_cache_drops_removed_caches(slot: CacheSlot, valid: bool, insert_error: bool) { - use reth_trie_common::bundle_state::{AccountStatus, BundleAccount, BundleState}; - let runtime = Runtime::test(); let execution_cache = PayloadExecutionCache::default(); let cache_to_save = diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index d30436c6df8..ebddc0b8123 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -811,7 +811,7 @@ where self.runtime.spawn_blocking_named("hash-post-state", move || { let _span = debug_span!( target: "engine::tree::payload_validator", - parent: parent_span, + parent: &parent_span, "hashed_post_state", ) .entered(); @@ -1014,7 +1014,7 @@ where self.runtime.spawn_blocking_named("payload-convert", move || { let _span = debug_span!( target: "engine::tree::payload_validator", - parent: parent_span, + parent: &parent_span, "convert_and_validate", ) .entered(); diff --git a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs index 0be5b608b84..046b3e79960 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs @@ -240,7 +240,7 @@ where let parent_span = tracing::Span::current(); let hashing_metrics = metrics.clone(); executor.spawn_blocking_named("trie-hashing", move || { - let _span = trace_span!(parent: parent_span, "run_hashing_task").entered(); + let _span = trace_span!(parent: &parent_span, "run_hashing_task").entered(); Self::run_hashing_task(updates, hashed_state_tx, hashing_metrics) }); diff --git a/crates/engine/tree/src/tree/tests.rs b/crates/engine/tree/src/tree/tests.rs index 91c338b77cc..699cd39ea55 100644 --- a/crates/engine/tree/src/tree/tests.rs +++ b/crates/engine/tree/src/tree/tests.rs @@ -45,7 +45,7 @@ use std::{ collections::BTreeMap, str::FromStr, sync::{ - mpsc::{Receiver, Sender}, + mpsc::{channel, Receiver, Sender}, Arc, }, time::Duration, @@ -122,7 +122,6 @@ struct TestChannel { impl TestChannel { /// Creates a new test channel fn spawn_channel() -> (Sender, Receiver, TestChannelHandle) { - use std::sync::mpsc::channel; let (original_tx, original_rx) = channel(); let (wrapped_tx, wrapped_rx) = channel(); let (release_tx, release_rx) = channel(); @@ -186,7 +185,6 @@ impl TestHarness { } fn with_config(chain_spec: Arc, tree_config: TreeConfig) -> Self { - use std::sync::mpsc::channel; let (action_tx, action_rx) = channel(); Self::with_persistence_channel_and_config(chain_spec, action_tx, action_rx, tree_config) } @@ -2257,7 +2255,6 @@ fn test_validate_block_synchronous_strategy_during_persistence() { let mut test_harness = ValidatorTestHarness::new(MAINNET.clone()); // Set up persistence action to force `Synchronous` strategy - use crate::tree::persistence_state::CurrentPersistenceAction; let persistence_action = CurrentPersistenceAction::SavingBlocks { highest: alloy_eips::NumHash::new(1, B256::random()), }; diff --git a/crates/era-downloader/src/client.rs b/crates/era-downloader/src/client.rs index e89f7f632ee..bc10074eecf 100644 --- a/crates/era-downloader/src/client.rs +++ b/crates/era-downloader/src/client.rs @@ -397,10 +397,12 @@ struct FileList { /// `extensions` are tried in order; pass them longest-first so `.ere` never matches inside `.erae`. fn extract_era_filename<'a>(line: &'a str, extensions: &[&str]) -> Option<&'a str> { for ext in extensions { + // The separator can be multibyte, so the name starts after its full UTF-8 width. if let Some(j) = line.find(ext) && - let Some(i) = line[..j].rfind(|c: char| !c.is_alphanumeric() && c != '-') + let Some((i, sep)) = + line[..j].char_indices().rfind(|&(_, c)| !c.is_alphanumeric() && c != '-') { - return Some(&line[i + 1..j + ext.len()]); + return Some(&line[i + sep.len_utf8()..j + ext.len()]); } } None @@ -473,6 +475,10 @@ mod tests { Some("mainnet-00600-a81ae85f.era1"); "era1 anchor" )] #[test_case("", &[".erae", ".ere"], None; "no era file on line")] + #[test_case( + "", &[".era1"], + Some("mainnet-00000-a81ae85f.era1"); "multibyte separator before name" + )] fn test_extract_era_filename(line: &str, exts: &[&str], expected: Option<&str>) { assert_eq!(extract_era_filename(line, exts), expected); } diff --git a/crates/era-utils/tests/it/history.rs b/crates/era-utils/tests/it/history.rs index 2a7955caa49..8acc5e0c578 100644 --- a/crates/era-utils/tests/it/history.rs +++ b/crates/era-utils/tests/it/history.rs @@ -1,7 +1,16 @@ use crate::{ClientWithFakeIndex, FileMeta, ITHACA_ERA_INDEX_URL}; use reqwest::{Client, Url}; use reth_db_common::init::init_genesis; -use reth_era::era1::types::execution::MAX_BLOCKS_PER_ERA1; +use reth_era::{ + e2s::types::{Entry, Header}, + era1::types::execution::MAX_BLOCKS_PER_ERA1, + ere::types::{ + execution::{ + COMPRESSED_BODY, COMPRESSED_HEADER, COMPRESSED_SLIM_RECEIPTS, TOTAL_DIFFICULTY, + }, + group::{DynamicBlockIndex, DYNAMIC_BLOCK_INDEX}, + }, +}; use reth_era_downloader::{EraClient, EraStream, EraStreamConfig}; use reth_era_utils::{export, import, Era1, Ere, ExportConfig}; use reth_etl::Collector; @@ -13,7 +22,7 @@ use reth_provider::{ }; use reth_stages_types::{StageCheckpoint, StageId}; use reth_static_file_types::StaticFileSegment; -use std::str::FromStr; +use std::{collections::HashMap, str::FromStr}; use tempfile::tempdir; const EXPORT_FIRST_BLOCK: u64 = 0; @@ -343,17 +352,6 @@ async fn test_ere_roundtrip_export_after_import() { /// Offsets are `i64`, relative to the index record, laid out per block as /// `[header, body, receipts, difficulty]` (the `noproofs` profile, so no proof component). fn assert_ere_index_offsets_resolve(path: &std::path::Path) { - use reth_era::{ - e2s::types::{Entry, Header}, - ere::types::{ - execution::{ - COMPRESSED_BODY, COMPRESSED_HEADER, COMPRESSED_SLIM_RECEIPTS, TOTAL_DIFFICULTY, - }, - group::{DynamicBlockIndex, DYNAMIC_BLOCK_INDEX}, - }, - }; - use std::collections::HashMap; - let bytes = fs::read(path).unwrap(); // Walk the TLV stream, recording each record's start position and type. diff --git a/crates/era/src/test_utils.rs b/crates/era/src/test_utils.rs index 0985e3f2138..983e659eb28 100644 --- a/crates/era/src/test_utils.rs +++ b/crates/era/src/test_utils.rs @@ -6,7 +6,8 @@ use crate::{ BlockTuple, CompressedBody, CompressedHeader, CompressedReceipts, TotalDifficulty, }, }; -use alloy_consensus::{Header, ReceiptWithBloom}; +use alloy_consensus::{BlockBody, Header, ReceiptWithBloom}; +use alloy_eips::eip4895::Withdrawals; use alloy_primitives::{Address, BlockNumber, Bytes, Log, LogData, B256, B64, U256}; use reth_ethereum_primitives::{Receipt, TxType}; @@ -113,10 +114,6 @@ pub(crate) fn create_sample_block(data_size: usize) -> BlockTuple { // Helper function to create a test block with compressed data pub(crate) fn create_test_block_with_compressed_data(number: BlockNumber) -> BlockTuple { - use alloy_consensus::{BlockBody, Header}; - use alloy_eips::eip4895::Withdrawals; - use alloy_primitives::{Address, Bytes, B256, B64, U256}; - // Create test header let header = Header { parent_hash: B256::default(), diff --git a/crates/ethereum/cli/src/interface.rs b/crates/ethereum/cli/src/interface.rs index e40d86b4080..d89003f4073 100644 --- a/crates/ethereum/cli/src/interface.rs +++ b/crates/ethereum/cli/src/interface.rs @@ -393,10 +393,11 @@ impl( stream_state: bool, on_state_update: &mut impl FnMut(StateUpdate), transaction: &Recovered, - commit: impl FnOnce(&TxResult) -> reth_evm::CommitChanges, + commit: impl FnOnce(&TxResult, evm2::TxStateChanges<'_>) -> reth_evm::CommitChanges, ) -> Result>, HandlerError> where T::Tx: Typed2718, @@ -172,7 +172,7 @@ where let mut update = StateUpdate::default(); let result = match evm.transact(transaction) { Ok(executed) => { - if commit(executed.result()).should_commit() { + if commit(executed.result(), executed.changes()).should_commit() { for changes in executed.account_changes() { block_state .commit_account_changes(changes, stream_state.then_some(&mut update)); @@ -664,7 +664,7 @@ mod tests { stream_state, &mut hook, &transaction, - |_| reth_evm::CommitChanges::Yes, + |_, _| reth_evm::CommitChanges::Yes, ) .unwrap() .unwrap(); diff --git a/crates/ethereum/evm/src/executor.rs b/crates/ethereum/evm/src/executor.rs index 60d8aa19bda..adc86abdb6f 100644 --- a/crates/ethereum/evm/src/executor.rs +++ b/crates/ethereum/evm/src/executor.rs @@ -306,7 +306,7 @@ where fn execute_transaction_with_commit_condition( &mut self, transaction: impl ExecutorTx, - commit: impl FnOnce(&evm2::TxResult) -> reth_evm::CommitChanges, + commit: impl FnOnce(&evm2::TxResult, evm2::TxStateChanges<'_>) -> reth_evm::CommitChanges, ) -> Result, BlockExecutionError> { let (transaction, tx) = transaction.into_parts(); self.set_transaction_block_access_index(); @@ -896,7 +896,10 @@ where fn execute_transaction_with_commit_condition( &mut self, transaction: impl ExecutorTx, - commit: impl FnOnce(&evm2::TxResult) -> reth_evm::CommitChanges, + commit: impl FnOnce( + &evm2::TxResult, + evm2::TxStateChanges<'_>, + ) -> reth_evm::CommitChanges, ) -> Result, BlockExecutionError> { self.initialize()?; let output = self.inner.execute_transaction_with_commit_condition(transaction, commit)?; @@ -1112,19 +1115,24 @@ mod tests { executor.apply_pre_execution_changes().expect("first segment pre-execution"); assert!(executor - .execute_transaction_with_commit_condition(transfer(from, first_target, 0), |_| { - CommitChanges::No - }) + .execute_transaction_with_commit_condition( + transfer(from, first_target, 0), + |_, changes| { + // The callback sees the uncommitted state changes before deciding. + assert_eq!(changes.account_info(&from).expect("sender changed").nonce, 1); + CommitChanges::No + } + ) .expect("discarded transaction") .is_none()); executor - .execute_transaction_with_commit_condition(transfer(from, first_target, 0), |_| { + .execute_transaction_with_commit_condition(transfer(from, first_target, 0), |_, _| { CommitChanges::Yes }) .expect("first segment transaction") .expect("transaction committed"); executor - .execute_transaction_with_commit_condition(transfer(from, second_target, 1), |_| { + .execute_transaction_with_commit_condition(transfer(from, second_target, 1), |_, _| { CommitChanges::Yes }) .expect("second segment transaction") diff --git a/crates/ethereum/node/src/engine_ssz_proxy.rs b/crates/ethereum/node/src/engine_ssz_proxy.rs index 8f5487b8ba3..5270d1ef2fe 100644 --- a/crates/ethereum/node/src/engine_ssz_proxy.rs +++ b/crates/ethereum/node/src/engine_ssz_proxy.rs @@ -1254,10 +1254,16 @@ fn problem_response( #[cfg(test)] mod tests { use super::*; - use alloy_rpc_types_engine::ssz_engine_types::{ - PayloadAttributesAmsterdam, PayloadAttributesCancun, PayloadAttributesParis, - PayloadAttributesShanghai, + use alloy_rpc_types_engine::{ + ssz_engine_types::{ + PayloadAttributesAmsterdam, PayloadAttributesCancun, PayloadAttributesParis, + PayloadAttributesShanghai, + }, + ExecutionPayloadV1, ExecutionPayloadV2, ExecutionPayloadV3, ForkchoiceUpdateError, }; + use reth_chainspec::{ChainSpecBuilder, ForkCondition}; + use reth_engine_primitives::BeaconForkChoiceUpdateError; + use reth_payload_primitives::PayloadBuilderError; use ssz::Encode; #[tokio::test] @@ -1311,7 +1317,6 @@ mod tests { #[test] fn payload_schema_bounds() { - use alloy_rpc_types_engine::{ExecutionPayloadV1, ExecutionPayloadV2, ExecutionPayloadV3}; let mut payload = ExecutionPayloadV3 { payload_inner: ExecutionPayloadV2 { payload_inner: ExecutionPayloadV1::from_block_unchecked( @@ -1440,9 +1445,6 @@ mod tests { #[tokio::test] async fn engine_errors_preserve_validation_semantics() { - use alloy_rpc_types_engine::ForkchoiceUpdateError; - use reth_engine_primitives::BeaconForkChoiceUpdateError; - use reth_payload_primitives::PayloadBuilderError; for (error, status, kind) in [ (EngineApiError::UnknownPayload, 404, "unknown-payload"), (EngineApiError::BlobRequestTooLarge { len: 129 }, 413, "request-too-large"), @@ -1495,7 +1497,6 @@ mod tests { #[test] fn payload_bodies_are_filtered_at_fork_boundaries() { - use reth_chainspec::{ChainSpecBuilder, ForkCondition}; let chain_spec = ChainSpecBuilder::default() .chain(1.into()) .genesis(Default::default()) diff --git a/crates/ethereum/node/tests/e2e/eth.rs b/crates/ethereum/node/tests/e2e/eth.rs index 9afda2b0161..212c6ef53b6 100644 --- a/crates/ethereum/node/tests/e2e/eth.rs +++ b/crates/ethereum/node/tests/e2e/eth.rs @@ -29,7 +29,10 @@ use reth_rpc_api::{TestingBuildBlockRequestV1, TestingBuildBlockResponseV1}; use reth_rpc_server_types::{RethRpcModule, RpcModuleSelection}; use reth_tasks::Runtime; use ssz::Encode; -use std::sync::Arc; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; const ENGINE_PAYLOADS_ROUTE: &str = "/engine/v1/payloads"; const ENGINE_FORKCHOICE_ROUTE: &str = "/engine/v1/forkchoice"; @@ -642,7 +645,6 @@ async fn test_engine_ssz_request_validation() -> eyre::Result<()> { #[tokio::test] async fn test_engine_ssz_custom_engine_and_middleware() -> eyre::Result<()> { - use std::sync::atomic::{AtomicUsize, Ordering}; let requests = Arc::new(AtomicUsize::new(0)); let observed = requests.clone(); let middleware = diff --git a/crates/ethereum/node/tests/e2e/rpc.rs b/crates/ethereum/node/tests/e2e/rpc.rs index a0608b0f56e..66bef3a2d7b 100644 --- a/crates/ethereum/node/tests/e2e/rpc.rs +++ b/crates/ethereum/node/tests/e2e/rpc.rs @@ -1,3 +1,4 @@ +use alloy_consensus::Header; use alloy_eips::{ eip2718::Encodable2718, eip7910::EthConfig, eip7928::BlockAccessList, BlockNumberOrTag, }; @@ -24,7 +25,10 @@ use alloy_rpc_types_trace::geth::{ }; use jsonrpsee::core::client::{ClientT, Subscription, SubscriptionClientT}; use rand::{rngs::StdRng, Rng, SeedableRng}; -use reth_chainspec::{EthChainSpec, EthereumHardfork}; +use reth_chainspec::{ + sepolia::{SEPOLIA_AMSTERDAM_TIMESTAMP, SEPOLIA_BPO2_TIMESTAMP}, + EthChainSpec, EthereumHardfork, SEPOLIA, +}; use reth_e2e_test_utils::{ receipt::PendingTransactionExt, test_chain_spec, test_chain_spec_builder, transaction::TransactionTestContext, wait::poll_until, wallet::Wallet, E2ETestSetupExt, @@ -38,10 +42,12 @@ use reth_node_core::{ }; use reth_node_ethereum::EthereumNode; use reth_payload_primitives::BuiltPayload; -use reth_primitives_traits::Block as _; +use reth_primitives_traits::{Block as _, SealedHeader, SignedTransaction}; +use reth_provider::CanonChainTracker; use reth_rpc_api::servers::AdminApiServer; use reth_rpc_server_types::RpcModuleSelection; use reth_tasks::Runtime; +use reth_transaction_pool::test_utils::TransactionGenerator; use std::{ net::{IpAddr, Ipv4Addr}, sync::Arc, @@ -64,9 +70,6 @@ alloy_sol_types::sol! { #[tokio::test] async fn test_rpc_shares_sender_recovery_cache_with_execution() -> eyre::Result<()> { - use reth_primitives_traits::SignedTransaction; - use reth_transaction_pool::test_utils::TransactionGenerator; - let chain_spec = test_chain_spec(EthereumHardfork::Cancun); let (node, _) = EthereumNode::test_setup(1, chain_spec).build_single().await?; let cache = node.inner.evm_config.sender_recovery_cache.as_ref().unwrap(); @@ -832,14 +835,6 @@ async fn test_eth_config() -> eyre::Result<()> { #[tokio::test] async fn test_sepolia_amsterdam_eth_config() -> eyre::Result<()> { - use alloy_consensus::Header; - use reth_chainspec::{ - sepolia::{SEPOLIA_AMSTERDAM_TIMESTAMP, SEPOLIA_BPO2_TIMESTAMP}, - SEPOLIA, - }; - use reth_primitives_traits::SealedHeader; - use reth_provider::CanonChainTracker; - let node_config = NodeConfig::test() .with_chain(SEPOLIA.clone()) .with_unused_ports() diff --git a/crates/ethereum/primitives/src/receipt.rs b/crates/ethereum/primitives/src/receipt.rs index cc27d283821..b18301f1d9e 100644 --- a/crates/ethereum/primitives/src/receipt.rs +++ b/crates/ethereum/primitives/src/receipt.rs @@ -206,8 +206,6 @@ mod tests { } #[test] fn check_receipt_root_optimism() { - use alloy_consensus::ReceiptWithBloom; - let logs = vec![Log { address: Address::ZERO, data: LogData::new_unchecked(vec![], Default::default()), diff --git a/crates/evm/evm/src/execute.rs b/crates/evm/evm/src/execute.rs index 63c946fc43c..bd0d90a041d 100644 --- a/crates/evm/evm/src/execute.rs +++ b/crates/evm/evm/src/execute.rs @@ -377,15 +377,22 @@ pub trait BlockExecutor: Sized { /// post-execution block changes. fn into_state(self) -> BundleState; - /// Executes a transaction, invokes `f` with the borrowed execution result, and commits - /// changes when `f` returns [`CommitChanges::Yes`]. + /// Executes a transaction, invokes `f` with the borrowed execution result and state changes, + /// and commits changes when `f` returns [`CommitChanges::Yes`]. + /// + /// The state changes are borrowed before the transaction is committed or discarded, so `f` + /// can read what the transaction wrote (balances, storage slots) to decide whether to keep it. fn execute_transaction_with_commit_condition( &mut self, transaction: impl ExecutorTx, - f: impl FnOnce(&evm2::TxResult<::EvmTypes>) -> CommitChanges, + f: impl FnOnce( + &evm2::TxResult<::EvmTypes>, + evm2::TxStateChanges<'_>, + ) -> CommitChanges, ) -> Result, BlockExecutionError> { let output = self.execute_transaction_without_commit(transaction)?; - if f(&output.result().result).should_commit() { + let result = output.result(); + if f(&result.result, result.pending_state.changes()).should_commit() { self.commit_transaction(output).map(Some) } else { Ok(None) @@ -417,7 +424,7 @@ pub trait BlockExecutor: Sized { transaction: impl ExecutorTx, f: impl FnOnce(&evm2::TxResult<::EvmTypes>), ) -> Result { - self.execute_transaction_with_commit_condition(transaction, |result| { + self.execute_transaction_with_commit_condition(transaction, |result, _| { f(result); CommitChanges::Yes }) @@ -624,13 +631,14 @@ pub trait BlockBuilder: Sized { /// Applies pre-execution block changes. fn apply_pre_execution_changes(&mut self) -> Result<(), BlockExecutionError>; - /// Executes a transaction, exposes its borrowed execution result to `f`, and saves it - /// for block assembly only if committed. + /// Executes a transaction, exposes its borrowed execution result and state changes to `f`, + /// and saves it for block assembly only if committed. fn execute_transaction_with_commit_condition( &mut self, tx: impl ExecutorTx, f: impl FnOnce( &evm2::TxResult<<::Evm as Evm>::EvmTypes>, + evm2::TxStateChanges<'_>, ) -> CommitChanges, ) -> Result, BlockExecutionError>; @@ -647,7 +655,7 @@ pub trait BlockBuilder: Sized { &mut self, tx: impl ExecutorTx, ) -> Result { - self.execute_transaction_with_commit_condition(tx, |_| CommitChanges::Yes) + self.execute_transaction_with_commit_condition(tx, |_, _| CommitChanges::Yes) .map(Option::unwrap_or_default) } @@ -787,6 +795,7 @@ where tx: impl ExecutorTx, f: impl FnOnce( &evm2::TxResult<<::Evm as Evm>::EvmTypes>, + evm2::TxStateChanges<'_>, ) -> CommitChanges, ) -> Result, BlockExecutionError> { let (tx_env, tx) = tx.into_parts(); diff --git a/crates/evm/execution-types/src/chain.rs b/crates/evm/execution-types/src/chain.rs index 601b83d5671..f40e4689c9f 100644 --- a/crates/evm/execution-types/src/chain.rs +++ b/crates/evm/execution-types/src/chain.rs @@ -558,8 +558,8 @@ pub(super) mod serde_bincode_compat { use alloy_rlp::Decodable; use core::marker::PhantomData; use reth_ethereum_primitives::EthPrimitives; - use reth_primitives_traits::{NodePrimitives, SealedBlock}; - use reth_trie_common::SortedTrieData; + use reth_primitives_traits::{NodePrimitives, RecoveredBlock, SealedBlock}; + use reth_trie_common::{LazyTrieData, SortedTrieData}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde_with::{DeserializeAs, SerializeAs}; @@ -642,9 +642,6 @@ pub(super) mod serde_bincode_compat { N: NodePrimitives, { fn from(value: Chain<'a, N>) -> Self { - use reth_primitives_traits::RecoveredBlock; - use reth_trie_common::LazyTrieData; - let hashed_state_map: BTreeMap<_, _> = value.hashed_state.into_iter().map(|(k, v)| (k, Arc::new(v.into()))).collect(); @@ -706,6 +703,7 @@ pub(super) mod serde_bincode_compat { #[cfg(test)] mod tests { use super::super::{serde_bincode_compat, Chain}; + use alloc::collections::BTreeMap; use arbitrary::Arbitrary; use rand::Rng; use reth_primitives_traits::RecoveredBlock; @@ -714,8 +712,6 @@ pub(super) mod serde_bincode_compat { #[test] fn test_chain_bincode_roundtrip() { - use alloc::collections::BTreeMap; - #[serde_as] #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] struct Data { diff --git a/crates/exex/exex/src/wal/storage.rs b/crates/exex/exex/src/wal/storage.rs index 2a66452f039..b83958057af 100644 --- a/crates/exex/exex/src/wal/storage.rs +++ b/crates/exex/exex/src/wal/storage.rs @@ -185,8 +185,9 @@ mod tests { Address, B256, U256, }; use evm2::{bytecode::Bytecode, evm::AccountInfo}; + use reth_ethereum_primitives::Block; use reth_exex_types::ExExNotification; - use reth_primitives_traits::Account; + use reth_primitives_traits::{Account, Block as _}; use reth_provider::Chain; use reth_testing_utils::generators::{self, random_block}; use reth_trie_common::{ @@ -196,7 +197,7 @@ mod tests { BranchNodeCompact, HashedPostState, HashedStorage, HashedStorageSorted, LazyTrieData, Nibbles, SortedTrieData, }; - use std::{collections::BTreeMap, fs::File, sync::Arc}; + use std::{collections::BTreeMap, fs::File, io::Write, sync::Arc}; #[test] fn test_roundtrip() -> eyre::Result<()> { @@ -310,8 +311,6 @@ mod tests { #[test] #[ignore] fn generate_test_wal() -> eyre::Result<()> { - use std::io::Write; - let notification = get_test_notification_data()?; // Serialize the notification @@ -337,9 +336,6 @@ mod tests { /// Helper function to generate deterministic test data for WAL tests fn get_test_notification_data( ) -> eyre::Result> { - use reth_ethereum_primitives::Block; - use reth_primitives_traits::Block as _; - // Create a block with a transaction let block = Block::default().seal_slow().try_recover()?; let block_number = block.header().number(); diff --git a/crates/firehose/src/executor.rs b/crates/firehose/src/executor.rs index 2a94abb9895..c470f0084f5 100644 --- a/crates/firehose/src/executor.rs +++ b/crates/firehose/src/executor.rs @@ -520,11 +520,15 @@ where fn execute_transaction_with_commit_condition( &mut self, transaction: impl ExecutorTx, - f: impl FnOnce(&evm2::TxResult<::EvmTypes>) -> CommitChanges, + f: impl FnOnce( + &evm2::TxResult<::EvmTypes>, + evm2::TxStateChanges<'_>, + ) -> CommitChanges, ) -> Result, BlockExecutionError> { let executed = self.execute_and_trace_transaction(transaction)?; - if !f(&executed.result.result().result).should_commit() { + let result = executed.result.result(); + if !f(&result.result, result.pending_state.changes()).should_commit() { // Preserve the invariant that every on_tx_start is paired with on_tx_end. let err = std::io::Error::other("transaction not committed"); self.inspector.tracer_mut().on_tx_end(None, Some(&err)); diff --git a/crates/net/discv4/src/config.rs b/crates/net/discv4/src/config.rs index 007d85ea794..3b3c2c95cf1 100644 --- a/crates/net/discv4/src/config.rs +++ b/crates/net/discv4/src/config.rs @@ -328,6 +328,8 @@ impl Discv4ConfigBuilder { #[cfg(test)] mod tests { use super::*; + use reth_net_nat::NatResolver; + use std::net::{IpAddr, Ipv4Addr}; #[test] fn test_config_builder() { @@ -343,9 +345,6 @@ mod tests { #[tokio::test] async fn test_resolve_external_ip_interval_uses_interval_at() { - use reth_net_nat::NatResolver; - use std::net::{IpAddr, Ipv4Addr}; - let ip_addr = IpAddr::V4(Ipv4Addr::new(192, 168, 1, 1)); // Create a config with external IP resolver diff --git a/crates/net/discv4/src/proto.rs b/crates/net/discv4/src/proto.rs index ee9968e9aaf..ec1cab8eba3 100644 --- a/crates/net/discv4/src/proto.rs +++ b/crates/net/discv4/src/proto.rs @@ -398,7 +398,7 @@ impl Decodable for EnrRequest { /// /// This packet is used to respond to an `ENRRequest` packet and includes the requested ENR along /// with the hash of the original request. -#[derive(Clone, Debug, Eq, PartialEq, RlpEncodable, RlpDecodable)] +#[derive(Clone, Debug, Eq, PartialEq, RlpEncodable)] pub struct EnrResponse { /// The hash of the `ENRRequest` packet being replied to. pub request_hash: B256, @@ -418,6 +418,24 @@ impl EnrResponse { } } +impl Decodable for EnrResponse { + // Manual implementation to satisfy EIP-8: additional list elements are ignored. + // + // See https://eips.ethereum.org/EIPS/eip-8 + fn decode(buf: &mut &[u8]) -> alloy_rlp::Result { + let mut payload = Header::decode_bytes(buf, true)?; + let request_hash = Decodable::decode(&mut payload)?; + + // `Enr::decode` applies the ENR size limit to its whole input, so it only gets the + // record's own bytes and not the additional list elements after it. + let mut rest = payload; + Header::decode_bytes(&mut rest, true)?; + let mut enr = &payload[..payload.len() - rest.len()]; + + Ok(Self { request_hash, enr: Decodable::decode(&mut enr)? }) + } +} + /// Represents a Ping packet. /// /// A [Ping packet](https://github.com/ethereum/devp2p/blob/master/discv4.md#ping-packet-0x01). @@ -611,10 +629,12 @@ mod tests { DEFAULT_DISCOVERY_PORT, SAFE_MAX_DATAGRAM_NEIGHBOUR_RECORDS, }; use alloy_primitives::hex; + use alloy_rlp::Decodable; use assert_matches::assert_matches; - use enr::EnrPublicKey; + use enr::{secp256k1::SecretKey, EnrPublicKey}; use rand_08::{thread_rng as rng, Rng, RngCore}; use reth_ethereum_forks::ForkHash; + use std::net::Ipv4Addr; #[test] fn test_endpoint_ipv_v4() { @@ -829,10 +849,6 @@ mod tests { #[test] fn encode_decode_enr_msg() { - use alloy_rlp::Decodable; - use enr::secp256k1::SecretKey; - use std::net::Ipv4Addr; - let mut rng = rand_08::rngs::OsRng; let key = SecretKey::new(&mut rng); let ip = Ipv4Addr::new(127, 0, 0, 1); @@ -867,10 +883,6 @@ mod tests { #[test] fn encode_known_rlp_enr() { - use alloy_rlp::Decodable; - use enr::{secp256k1::SecretKey, EnrPublicKey}; - use std::net::Ipv4Addr; - let valid_record = hex!( "f884b8407098ad865b00a582051940cb9cf36836572411a47278783077011599ed5cd16b76f2635f4e234738f30813a89eb9137e3e3df5266e3a1f11df72ecf1145ccb9c01826964827634826970847f00000189736563703235366b31a103ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd31388375647082765f" ); @@ -901,9 +913,6 @@ mod tests { // #[test] fn decode_enr_rlp() { - use enr::secp256k1::SecretKey; - use std::net::Ipv4Addr; - let valid_record = hex!( "f884b8407098ad865b00a582051940cb9cf36836572411a47278783077011599ed5cd16b76f2635f4e234738f30813a89eb9137e3e3df5266e3a1f11df72ecf1145ccb9c01826964827634826970847f00000189736563703235366b31a103ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd31388375647082765f" ); @@ -950,9 +959,6 @@ mod tests { // #[test] fn encode_decode_enr_rlp() { - use enr::{secp256k1::SecretKey, EnrPublicKey}; - use std::net::Ipv4Addr; - let key = SecretKey::new(&mut rand_08::rngs::OsRng); let ip = Ipv4Addr::new(127, 0, 0, 1); let tcp = 3000; @@ -1079,5 +1085,25 @@ mod tests { ]; assert_matches!(decoded.msg, Message::Neighbours(Neighbours { nodes, expire: 1136239445 }) if nodes == expected_nodes); } + + // checks that junk data at the end of the packet is discarded according to eip-8, even + // when the record and the junk together exceed the ENR size limit. + #[test] + fn eip8_decode_enr_response() { + let key = SecretKey::new(&mut rand_08::rngs::OsRng); + let enr = Enr::builder().build(&key).unwrap(); + let request_hash = B256::random(); + let junk = Bytes::from(vec![0xab; 300]); + + let mut buf = Vec::new(); + let payload_length = request_hash.length() + enr.length() + junk.length(); + alloy_rlp::Header { list: true, payload_length }.encode(&mut buf); + request_hash.encode(&mut buf); + enr.encode(&mut buf); + junk.encode(&mut buf); + + let decoded = EnrResponse::decode(&mut buf.as_slice()).unwrap(); + assert_eq!(decoded, EnrResponse { request_hash, enr }); + } } } diff --git a/crates/net/downloaders/src/test_utils/bodies_client.rs b/crates/net/downloaders/src/test_utils/bodies_client.rs index e0936d4cd01..8e818814c2a 100644 --- a/crates/net/downloaders/src/test_utils/bodies_client.rs +++ b/crates/net/downloaders/src/test_utils/bodies_client.rs @@ -59,11 +59,9 @@ impl TestBodiesClient { /// This will only return true if `empty_response_mod` is `Some`, and `times_requested % /// empty_response_mod == 0`. pub(crate) fn should_respond_empty(&self) -> bool { - if let Some(empty_response_mod) = self.empty_response_mod { + self.empty_response_mod.is_some_and(|empty_response_mod| { self.times_requested.load(Ordering::Relaxed).is_multiple_of(empty_response_mod) - } else { - false - } + }) } } diff --git a/crates/net/eth-wire-types/src/message.rs b/crates/net/eth-wire-types/src/message.rs index 8ebf0f6b476..55120104f38 100644 --- a/crates/net/eth-wire-types/src/message.rs +++ b/crates/net/eth-wire-types/src/message.rs @@ -890,9 +890,10 @@ mod tests { use crate::{ message::RequestPair, BlockAccessLists, EthMessage, EthMessageID, EthNetworkPrimitives, EthVersion, GetBlockAccessLists, GetNodeData, NodeData, ProtocolMessage, - RawCapabilityMessage, + RawCapabilityMessage, Status, StatusMessage, }; - use alloy_primitives::hex; + use alloy_hardforks::{ForkHash, ForkId}; + use alloy_primitives::{hex, B256, U256}; use alloy_rlp::{Decodable, Encodable, Error}; use reth_ethereum_primitives::BlockBody; @@ -1111,10 +1112,6 @@ mod tests { #[test] fn decode_status_success() { - use crate::{Status, StatusMessage}; - use alloy_hardforks::{ForkHash, ForkId}; - use alloy_primitives::{B256, U256}; - let status = Status { version: EthVersion::Eth68, chain: alloy_chains::Chain::mainnet(), diff --git a/crates/net/eth-wire/src/handshake.rs b/crates/net/eth-wire/src/handshake.rs index 1dcf9dd9825..45fb2ac5439 100644 --- a/crates/net/eth-wire/src/handshake.rs +++ b/crates/net/eth-wire/src/handshake.rs @@ -92,7 +92,7 @@ where EthMessage::Status(status), )) .into(); - unauth.send(status_msg).await.map_err(EthStreamError::from)?; + unauth.send(status_msg).await?; // Receive peer's response let their_msg_res = unauth.next().await; @@ -100,19 +100,13 @@ where Some(Ok(msg)) => msg, Some(Err(e)) => return Err(EthStreamError::from(e)), None => { - unauth - .disconnect(DisconnectReason::DisconnectRequested) - .await - .map_err(EthStreamError::from)?; + unauth.disconnect(DisconnectReason::DisconnectRequested).await?; return Err(EthStreamError::EthHandshakeError(EthHandshakeError::NoResponse)); } }; if their_msg.len() > MAX_MESSAGE_SIZE { - unauth - .disconnect(DisconnectReason::ProtocolBreach) - .await - .map_err(EthStreamError::from)?; + unauth.disconnect(DisconnectReason::ProtocolBreach).await?; return Err(EthStreamError::MessageTooBig(their_msg.len())); } @@ -124,10 +118,7 @@ where Ok(status) => status, Err(err) => { debug!("decode error in eth handshake: msg={their_msg:x}"); - unauth - .disconnect(DisconnectReason::ProtocolBreach) - .await - .map_err(EthStreamError::from)?; + unauth.disconnect(DisconnectReason::ProtocolBreach).await?; return Err(EthStreamError::InvalidMessage(err)); } }; @@ -135,10 +126,7 @@ where trace!("Validating incoming ETH status from peer"); if status.genesis() != their_status_message.genesis() { - unauth - .disconnect(DisconnectReason::ProtocolBreach) - .await - .map_err(EthStreamError::from)?; + unauth.disconnect(DisconnectReason::ProtocolBreach).await?; return Err(EthHandshakeError::MismatchedGenesis( GotExpected { expected: status.genesis(), got: their_status_message.genesis() } .into(), @@ -147,10 +135,7 @@ where } if status.version() != their_status_message.version() { - unauth - .disconnect(DisconnectReason::ProtocolBreach) - .await - .map_err(EthStreamError::from)?; + unauth.disconnect(DisconnectReason::ProtocolBreach).await?; return Err(EthHandshakeError::MismatchedProtocolVersion(GotExpected { got: their_status_message.version(), expected: status.version(), @@ -159,10 +144,7 @@ where } if *status.chain() != *their_status_message.chain() { - unauth - .disconnect(DisconnectReason::ProtocolBreach) - .await - .map_err(EthStreamError::from)?; + unauth.disconnect(DisconnectReason::ProtocolBreach).await?; return Err(EthHandshakeError::MismatchedChain(GotExpected { got: *their_status_message.chain(), expected: *status.chain(), @@ -174,10 +156,7 @@ where if let StatusMessage::Legacy(s) = &their_status_message && s.total_difficulty.bit_len() > 160 { - unauth - .disconnect(DisconnectReason::ProtocolBreach) - .await - .map_err(EthStreamError::from)?; + unauth.disconnect(DisconnectReason::ProtocolBreach).await?; return Err(EthHandshakeError::TotalDifficultyBitLenTooLarge { got: s.total_difficulty.bit_len(), maximum: 160, @@ -190,10 +169,7 @@ where .validate(their_status_message.forkid()) .map_err(EthHandshakeError::InvalidFork) { - unauth - .disconnect(DisconnectReason::ProtocolBreach) - .await - .map_err(EthStreamError::from)?; + unauth.disconnect(DisconnectReason::ProtocolBreach).await?; return Err(err.into()); } diff --git a/crates/net/eth-wire/src/hello.rs b/crates/net/eth-wire/src/hello.rs index 83209131915..b12d0aa5334 100644 --- a/crates/net/eth-wire/src/hello.rs +++ b/crates/net/eth-wire/src/hello.rs @@ -1,5 +1,5 @@ use crate::{Capability, EthVersion, ProtocolVersion}; -use alloy_rlp::{RlpDecodable, RlpEncodable}; +use alloy_rlp::{Decodable, Header, RlpEncodable}; use reth_codecs::add_arbitrary_tests; use reth_network_peers::PeerId; use reth_primitives_traits::constants::RETH_CLIENT_VERSION; @@ -111,12 +111,13 @@ impl HelloMessageWithProtocols { } } -// TODO: determine if we should allow for the extra fields at the end like EIP-706 suggests /// Raw rlpx protocol message used in the `p2p` handshake, containing information about the /// supported `RLPx` protocol version and capabilities. /// +/// Decoding ignores list elements after `id`, which the spec reserves for future versions (EIP-8). +/// /// See also -#[derive(Clone, Debug, PartialEq, Eq, RlpEncodable, RlpDecodable)] +#[derive(Clone, Debug, PartialEq, Eq, RlpEncodable)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))] #[add_arbitrary_tests(rlp)] @@ -136,6 +137,19 @@ pub struct HelloMessage { // === impl HelloMessage === +impl Decodable for HelloMessage { + fn decode(buf: &mut &[u8]) -> alloy_rlp::Result { + let mut payload = Header::decode_bytes(buf, true)?; + Ok(Self { + protocol_version: Decodable::decode(&mut payload)?, + client_version: Decodable::decode(&mut payload)?, + capabilities: Decodable::decode(&mut payload)?, + port: Decodable::decode(&mut payload)?, + id: Decodable::decode(&mut payload)?, + }) + } +} + impl HelloMessage { /// Starts a new `HelloMessageBuilder` /// @@ -232,7 +246,7 @@ mod tests { p2pstream::P2PMessage, Capability, EthVersion, HelloMessage, HelloMessageWithProtocols, ProtocolVersion, }; - use alloy_rlp::{Decodable, Encodable, EMPTY_STRING_CODE}; + use alloy_rlp::{Decodable, Encodable, Header, EMPTY_STRING_CODE}; use reth_network_peers::pk2id; use secp256k1::{SecretKey, SECP256K1}; @@ -308,4 +322,35 @@ mod tests { // zero is encoded as 0x80, the empty string code in RLP assert_eq!(hello_encoded[0], EMPTY_STRING_CODE); } + + #[test] + fn hello_decode_ignores_extra_fields() { + // EIP-8: additional list elements must be ignored + let secret_key = SecretKey::new(&mut rand_08::thread_rng()); + let hello = HelloMessage { + protocol_version: ProtocolVersion::V5, + client_version: "reth/0.1.0".to_string(), + capabilities: vec![Capability::new_static("eth", EthVersion::Eth68 as usize)], + port: 30303, + id: pk2id(&secret_key.public_key(SECP256K1)), + }; + + let mut payload = Vec::new(); + hello.protocol_version.encode(&mut payload); + hello.client_version.encode(&mut payload); + hello.capabilities.encode(&mut payload); + hello.port.encode(&mut payload); + hello.id.encode(&mut payload); + 42u8.encode(&mut payload); + "future".encode(&mut payload); + + let mut encoded = Vec::new(); + Header { list: true, payload_length: payload.len() }.encode(&mut encoded); + encoded.extend_from_slice(&payload); + encoded.push(0xff); + + let mut buf = encoded.as_slice(); + assert_eq!(HelloMessage::decode(&mut buf).unwrap(), hello); + assert_eq!(buf, [0xff]); + } } diff --git a/crates/net/eth-wire/src/multiplex.rs b/crates/net/eth-wire/src/multiplex.rs index 63c29e23ed4..d76111c5aeb 100644 --- a/crates/net/eth-wire/src/multiplex.rs +++ b/crates/net/eth-wire/src/multiplex.rs @@ -972,7 +972,7 @@ impl ProtocolStream { /// Sends the message to the satellite stream. fn send_raw(&self, msg: BytesMut) -> Result<(), P2PStreamError> { - let msg = self.unmask_id(msg).map_err(P2PStreamError::from)?; + let msg = self.unmask_id(msg)?; self.to_satellite.try_send(msg) } } diff --git a/crates/net/network/src/eth_requests.rs b/crates/net/network/src/eth_requests.rs index 13c91ba1957..d597c67dff1 100644 --- a/crates/net/network/src/eth_requests.rs +++ b/crates/net/network/src/eth_requests.rs @@ -883,6 +883,7 @@ mod tests { eip7594::{BlobCellMask, BlobTransactionSidecarVariant, Cell}, }; use alloy_primitives::{keccak256, Address, Log, LogData, TxHash, B128, U256}; + use alloy_rlp::{Decodable, Encodable}; use reth_eth_wire::{message::RequestPair, EthMessage, EthStreamInner, EthVersion}; use reth_ethereum_primitives::Receipt; use reth_network_api::test_utils::PeersHandle; @@ -1047,8 +1048,6 @@ mod tests { #[test_case(127)] #[tokio::test] async fn get_cells_request_uses_little_endian_wire_mask(index: u32) { - use alloy_rlp::{Decodable, Encodable}; - let (peers_tx, _) = mpsc::unbounded_channel(); let (_incoming_tx, incoming_rx) = mpsc::channel(1); let numeric_mask = 1u128 << index; diff --git a/crates/net/network/src/fetch/mod.rs b/crates/net/network/src/fetch/mod.rs index 5488935d2cc..63605a555e6 100644 --- a/crates/net/network/src/fetch/mod.rs +++ b/crates/net/network/src/fetch/mod.rs @@ -865,12 +865,16 @@ mod tests { use crate::{peers::PeersManager, PeersConfig}; use alloy_consensus::Header; use alloy_primitives::{Bytes, B512}; + use futures::task::noop_waker; use reth_eth_wire::Capability; use reth_eth_wire_types::snap::{ AccountRangeMessage, BlockAccessListsMessage, ByteCodesMessage, GetAccountRangeMessage, StorageRangesMessage, }; - use std::future::poll_fn; + use std::{ + future::poll_fn, + task::{Context, Poll}, + }; use test_case::test_case; #[tokio::test(flavor = "multi_thread")] @@ -1864,9 +1868,6 @@ mod tests { #[tokio::test] async fn test_wakes_when_eth71_peer_connects() { - use futures::task::noop_waker; - use std::task::{Context, Poll}; - let manager = PeersManager::new(PeersConfig::default()); let mut fetcher = StateFetcher::::new(manager.handle(), Default::default()); @@ -1925,9 +1926,6 @@ mod tests { #[tokio::test] async fn test_optional_bal_request_rejected_without_eth71_peer() { - use futures::task::noop_waker; - use std::task::{Context, Poll}; - let manager = PeersManager::new(PeersConfig::default()); let mut fetcher = StateFetcher::::new(manager.handle(), Default::default()); @@ -1965,9 +1963,6 @@ mod tests { #[tokio::test] async fn test_optional_bal_request_waits_for_busy_eth71_peer() { - use futures::task::noop_waker; - use std::task::{Context, Poll}; - let manager = PeersManager::new(PeersConfig::default()); let mut fetcher = StateFetcher::::new(manager.handle(), Default::default()); @@ -2005,9 +2000,6 @@ mod tests { #[tokio::test] async fn test_queued_optional_bal_request_rejected_after_eth71_disconnect() { - use futures::task::noop_waker; - use std::task::{Context, Poll}; - let manager = PeersManager::new(PeersConfig::default()); let mut fetcher = StateFetcher::::new(manager.handle(), Default::default()); @@ -2118,9 +2110,6 @@ mod tests { #[tokio::test] async fn test_snap_request_rejected_without_snap_peer() { - use futures::task::noop_waker; - use std::task::{Context, Poll}; - let manager = PeersManager::new(PeersConfig::default()); let mut fetcher = StateFetcher::::new(manager.handle(), Default::default()); @@ -2216,9 +2205,6 @@ mod tests { #[tokio::test] async fn test_queued_snap_request_rejected_after_last_peer_disconnects() { - use futures::task::noop_waker; - use std::task::{Context, Poll}; - let manager = PeersManager::new(PeersConfig::default()); let mut fetcher = StateFetcher::::new(manager.handle(), Default::default()); diff --git a/crates/net/network/src/peers.rs b/crates/net/network/src/peers.rs index 44fe868dfa7..10747d417c6 100644 --- a/crates/net/network/src/peers.rs +++ b/crates/net/network/src/peers.rs @@ -149,9 +149,11 @@ impl PeersManager { HashSet::with_capacity_and_hasher(trusted_nodes.len(), Default::default()); for trusted_peer in &trusted_nodes { + // Tracked even if resolution fails, otherwise the resolver's later results for this + // peer are discarded as revoked. + trusted_peer_ids.insert(trusted_peer.id); match trusted_peer.resolve_blocking() { Ok(NodeRecord { address, tcp_port, udp_port, id }) => { - trusted_peer_ids.insert(id); peers.entry(id).or_insert_with(|| { Peer::trusted(PeerAddr::new_with_ports(address, tcp_port, Some(udp_port))) }); @@ -190,6 +192,14 @@ impl PeersManager { }); } + // A configured trusted peer that failed to resolve has no trusted entry yet, so a persisted + // or basic record for it must be promoted to keep the id set and the kind in sync. + for id in &trusted_peer_ids { + if let Some(peer) = peers.get_mut(id) { + peer.kind = PeerKind::Trusted; + } + } + trace!(target: "net::peers", trusted_peers=?trusted_peer_ids, "Initialized peers manager"); Self { @@ -467,6 +477,9 @@ impl PeersManager { let mut peer = Peer::with_state(PeerAddr::from_tcp(addr), PeerConnectionState::In); peer.mark_connected(); peer.remove_after_disconnect = true; + if is_trusted { + peer.kind = PeerKind::Trusted; + } entry.insert(peer); self.queued_actions.push_back(PeerAction::PeerAdded(peer_id)); } @@ -918,9 +931,11 @@ impl PeersManager { } /// Adds a trusted peer that may use a hostname instead of an IP address. + /// + /// An already tracked peer is marked as trusted right away. pub(crate) fn add_trusted_peer_node(&mut self, trusted: TrustedPeer) { let peer_id = trusted.id; - self.trusted_peer_ids.insert(peer_id); + self.add_trusted_peer_id(peer_id); self.trusted_peers_resolver.remove(peer_id); self.trusted_peers_resolver.trusted_peers.push(trusted); self.trusted_peers_resolver.interval.reset_immediately(); @@ -1274,6 +1289,13 @@ impl PeersManager { ); if let Some(peer) = self.peers.get_mut(&peer_id) { + // The peer may have been added as basic by discovery or an inbound session before the + // first successful resolution. + peer.kind = PeerKind::Trusted; + if peer.state.is_incoming() { + // the resolved address replaces the inbound one, see `add_peer_kind` + peer.remove_after_disconnect = false; + } if peer.addr != new_addr { peer.addr = new_addr; trace!(target: "net::peers", ?peer_id, addr=?peer.addr, "Updated resolved trusted peer address"); @@ -1623,6 +1645,7 @@ mod tests { session::PendingSessionHandshakeError, PeersConfig, }; + use reth_net_banlist::IpFilter; struct PeerActionFuture<'a> { peers: &'a mut PeersManager, @@ -3604,7 +3627,6 @@ mod tests { #[tokio::test] async fn test_ip_filter_blocks_inbound_connection() { - use reth_net_banlist::IpFilter; use std::net::IpAddr; // Create a filter that only allows 192.168.0.0/16 @@ -3623,7 +3645,6 @@ mod tests { #[tokio::test] async fn test_ip_filter_blocks_outbound_connection() { - use reth_net_banlist::IpFilter; use std::net::SocketAddr; // Create a filter that only allows 192.168.0.0/16 @@ -3647,7 +3668,6 @@ mod tests { #[tokio::test] async fn test_ip_filter_ipv6() { - use reth_net_banlist::IpFilter; use std::net::IpAddr; // Create a filter that only allows IPv6 range 2001:db8::/32 @@ -3666,7 +3686,6 @@ mod tests { #[tokio::test] async fn test_ip_filter_multiple_ranges() { - use reth_net_banlist::IpFilter; use std::net::IpAddr; // Create a filter that allows multiple ranges @@ -3687,7 +3706,6 @@ mod tests { #[tokio::test] async fn test_ip_filter_no_restriction() { - use reth_net_banlist::IpFilter; use std::net::IpAddr; // Create a filter with no restrictions (allow all) @@ -3753,6 +3771,57 @@ mod tests { assert_eq!(peer.addr.tcp().ip(), "10.0.0.1".parse::().unwrap()); } + #[tokio::test] + async fn test_configured_trusted_peer_resolves_after_startup_failure() { + let peer_id = PeerId::random(); + let trusted = TrustedPeer::new(Host::Domain("example.invalid".into()), 30303, peer_id); + let mut manager = PeersManager::new(PeersConfig::test().with_trusted_nodes(vec![trusted])); + assert!(!manager.peers.contains_key(&peer_id)); + + let addr = "10.0.0.1:30303".parse::().unwrap(); + manager.on_resolved_peer(peer_id, NodeRecord::new(addr, peer_id)); + let peer = manager.peers.get(&peer_id).expect("peer should be added after resolution"); + assert!(peer.is_trusted()); + assert_eq!(peer.addr.tcp(), addr); + } + + #[tokio::test] + async fn test_unresolved_trusted_peer_promotes_persisted_entry() { + let peer_id = PeerId::random(); + let trusted = TrustedPeer::new(Host::Domain("example.invalid".into()), 30303, peer_id); + let mut config = PeersConfig::test().with_trusted_nodes(vec![trusted]); + config.persisted_peers.push(PersistedPeerInfo { + record: NodeRecord::new("10.0.0.1:30303".parse().unwrap(), peer_id), + kind: PeerKind::Basic, + fork_id: None, + reputation: 0, + }); + let manager = PeersManager::new(config); + assert!(manager.trusted_peer_ids.contains(&peer_id)); + assert!(manager.peers.get(&peer_id).unwrap().is_trusted()); + } + + #[tokio::test] + async fn test_unresolved_trusted_peer_inbound_then_resolved() { + let peer_id = PeerId::random(); + let trusted = TrustedPeer::new(Host::Domain("example.invalid".into()), 30303, peer_id); + let mut manager = PeersManager::new( + PeersConfig::test().with_trusted_nodes(vec![trusted]).with_trusted_nodes_only(true), + ); + + let inbound = "10.0.0.1:41234".parse::().unwrap(); + assert!(manager.on_incoming_pending_session(inbound.ip()).is_ok()); + manager.on_incoming_session_established(peer_id, inbound); + assert!(manager.peers.get(&peer_id).unwrap().is_trusted()); + + let addr = "10.0.0.1:30303".parse::().unwrap(); + manager.on_resolved_peer(peer_id, NodeRecord::new(addr, peer_id)); + let peer = manager.peers.get(&peer_id).unwrap(); + assert!(peer.is_trusted()); + assert!(!peer.remove_after_disconnect); + assert_eq!(peer.addr.tcp(), addr); + } + #[tokio::test] async fn test_removed_trusted_peer_ignores_late_resolution() { let peer_id = PeerId::random(); @@ -3779,6 +3848,24 @@ mod tests { assert!(!manager.trusted_peer_ids.contains(&peer_id)); } + #[tokio::test] + async fn test_resolution_promotes_basic_trusted_peer() { + let peer_id = PeerId::random(); + let trusted = TrustedPeer { + host: url::Host::Domain("example.invalid".to_string()), + tcp_port: 30303, + udp_port: 30303, + id: peer_id, + }; + let addr = "10.0.0.1:30303".parse::().unwrap(); + + let mut manager = PeersManager::default(); + manager.add_trusted_peer_node(trusted); + manager.add_peer(peer_id, PeerAddr::from_tcp(addr), None); + manager.on_resolved_peer(peer_id, NodeRecord::new(addr, peer_id)); + assert!(manager.peers[&peer_id].is_trusted()); + } + #[tokio::test] async fn test_rotation_disconnects_eligible_peer() { let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 1, 2)), 8008); @@ -4338,7 +4425,12 @@ mod tests { if !register_first { peers.add_trusted_peer_node(trusted); } - assert_eq!(peers.peers[&peer_id].kind, PeerKind::Basic); + // A peer tracked before registration is promoted right away, one discovered afterwards + // only on resolution. + assert_eq!( + peers.peers[&peer_id].kind, + if register_first { PeerKind::Basic } else { PeerKind::Trusted } + ); peers.queued_actions.clear(); peers.fill_outbound_slots(); assert!( diff --git a/crates/net/network/src/session/active.rs b/crates/net/network/src/session/active.rs index 0f854053606..2219c75ed7f 100644 --- a/crates/net/network/src/session/active.rs +++ b/crates/net/network/src/session/active.rs @@ -993,12 +993,11 @@ impl Future for ActiveSession { // Check if we should send a range update based on block height changes while interval.poll_tick(cx).is_ready() { let current_latest = this.local_range_info.latest(); - let should_send = if let Some(last_sent) = this.last_sent_latest_block { - // Only send if block height has advanced by at least one epoch (32 blocks) + // The first update is always sent; after that, only send if block height has + // advanced by at least one epoch (32 blocks). + let should_send = this.last_sent_latest_block.is_none_or(|last_sent| { current_latest.saturating_sub(last_sent) >= EPOCH_SLOTS - } else { - true // First update, always send - }; + }); if should_send { this.queued_outgoing.push_back( @@ -1325,7 +1324,7 @@ mod tests { use super::*; use crate::session::{handle::PendingSessionEvent, start_pending_incoming_session}; use alloy_eips::eip2124::ForkFilter; - use alloy_primitives::B256; + use alloy_primitives::{B128, B256}; use futures::task::noop_waker; use reth_chainspec::MAINNET; use reth_ecies::stream::ECIESStream; @@ -2035,8 +2034,6 @@ mod tests { #[test] fn eth72_coalescing_preserves_cell_masks() { - use alloy_primitives::B128; - let masks = [None, Some(B128::repeat_byte(0xff)), Some(B128::from(1u128)), Some(B128::from(2u128))]; for existing_mask in masks { diff --git a/crates/net/peers/src/node_record.rs b/crates/net/peers/src/node_record.rs index ecd61b720d7..2d148169732 100644 --- a/crates/net/peers/src/node_record.rs +++ b/crates/net/peers/src/node_record.rs @@ -17,6 +17,7 @@ use serde_with::{DeserializeFromStr, SerializeDisplay}; #[cfg(feature = "secp256k1")] use enr::Enr; +use url::{Host, Url}; /// Represents an ENR in discovery. /// @@ -172,8 +173,6 @@ impl FromStr for NodeRecord { type Err = NodeRecordParseError; fn from_str(s: &str) -> Result { - use url::{Host, Url}; - let url = Url::parse(s).map_err(|e| NodeRecordParseError::InvalidUrl(e.to_string()))?; let address = match url.host() { diff --git a/crates/node/builder/src/launch/common.rs b/crates/node/builder/src/launch/common.rs index b8d2a026e07..d20c5fce57e 100644 --- a/crates/node/builder/src/launch/common.rs +++ b/crates/node/builder/src/launch/common.rs @@ -96,6 +96,7 @@ use tokio::sync::{ }; use futures::{future::Either, stream, Stream, StreamExt}; +use reth_engine_primitives::ConsensusEngineEvent; use reth_node_ethstats::EthStatsService; use reth_node_events::{cl::ConsensusLayerHealthEvents, node::NodeEvent}; @@ -1315,7 +1316,6 @@ where let task_executor = self.task_executor().clone(); task_executor.spawn_task(async move { while let Some(event) = engine_events.next().await { - use reth_engine_primitives::ConsensusEngineEvent; match event { ConsensusEngineEvent::ForkBlockAdded(executed, duration) | ConsensusEngineEvent::CanonicalBlockAdded(executed, duration) => { diff --git a/crates/node/builder/src/launch/invalid_block_hook.rs b/crates/node/builder/src/launch/invalid_block_hook.rs index ea2ae4b1fb7..280c8676f9e 100644 --- a/crates/node/builder/src/launch/invalid_block_hook.rs +++ b/crates/node/builder/src/launch/invalid_block_hook.rs @@ -5,7 +5,8 @@ use alloy_consensus::TxEnvelope; use alloy_rpc_types::{Block, Header, Receipt, Transaction, TransactionRequest}; use eyre::OptionExt; use reth_chainspec::EthChainSpec; -use reth_engine_primitives::InvalidBlockHook; +use reth_engine_primitives::{InvalidBlockHook, InvalidBlockHooks, NoopInvalidBlockHook}; +use reth_invalid_block_hooks::InvalidBlockWitnessHook; use reth_node_api::{FullNodeComponents, NodeTypes}; use reth_node_core::{ args::InvalidBlockHookType, @@ -75,9 +76,6 @@ where P: reth_provider::StateProviderFactory + reth_provider::ChainSpecProvider + Clone + 'static, E: reth_evm::ConfigureEvm + 'static, { - use reth_engine_primitives::{InvalidBlockHooks, NoopInvalidBlockHook}; - use reth_invalid_block_hooks::InvalidBlockWitnessHook; - let Some(ref hook) = config.debug.invalid_block_hook else { return Ok(Box::new(NoopInvalidBlockHook::default())) }; diff --git a/crates/node/core/src/args/trace.rs b/crates/node/core/src/args/trace.rs index f5f8d82a778..002a97b9d13 100644 --- a/crates/node/core/src/args/trace.rs +++ b/crates/node/core/src/args/trace.rs @@ -127,12 +127,15 @@ impl DefaultTraceValues { /// CLI arguments for configuring `Opentelemetry` trace and logs export. #[derive(Debug, Clone, Parser)] pub struct TraceArgs { - /// Enable `Opentelemetry` tracing export to an OTLP endpoint. + /// Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. /// /// If no value provided, defaults based on protocol: /// - HTTP: `http://localhost:4318/v1/traces` /// - gRPC: `http://localhost:4317` /// + /// A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without + /// a collector. Use a separate file per process. The network protocol is ignored for files. + /// /// Example: --tracing-otlp=http://collector:4318/v1/traces #[arg( long = "tracing-otlp", diff --git a/crates/payload/basic/src/lib.rs b/crates/payload/basic/src/lib.rs index 766a0438c12..6426c9933d9 100644 --- a/crates/payload/basic/src/lib.rs +++ b/crates/payload/basic/src/lib.rs @@ -169,14 +169,12 @@ where let parent_header = if parent_hash.is_zero() { // Use latest header for genesis block case self.client - .latest_header() - .map_err(PayloadBuilderError::from)? + .latest_header()? .ok_or_else(|| PayloadBuilderError::MissingParentHeader(B256::ZERO))? } else { // Fetch specific header by hash self.client - .sealed_header_by_hash(parent_hash) - .map_err(PayloadBuilderError::from)? + .sealed_header_by_hash(parent_hash)? .ok_or_else(|| PayloadBuilderError::MissingParentHeader(parent_hash))? }; @@ -1058,11 +1056,7 @@ impl fmt::Debug for MissingPayloadBehaviour { /// This compares the total fees of the blocks, higher is better. #[inline(always)] pub fn is_better_payload(best_payload: Option<&T>, new_fees: U256) -> bool { - if let Some(best_payload) = best_payload { - new_fees > best_payload.fees() - } else { - true - } + best_payload.is_none_or(|best_payload| new_fees > best_payload.fees()) } /// Returns the duration until the given unix timestamp in seconds. diff --git a/crates/payload/primitives/src/traits.rs b/crates/payload/primitives/src/traits.rs index 2ad71729a76..bc1c22f8843 100644 --- a/crates/payload/primitives/src/traits.rs +++ b/crates/payload/primitives/src/traits.rs @@ -10,6 +10,7 @@ use either::Either; use reth_execution_types::BlockExecutionOutput; use reth_primitives_traits::{NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader}; use reth_trie_common::{updates::TrieUpdates, HashedPostState}; +use sha2::Digest; /// Represents an executed block for payload building purposes. /// @@ -195,7 +196,6 @@ pub fn payload_id( parent: &B256, attributes: &alloy_rpc_types_engine::PayloadAttributes, ) -> PayloadId { - use sha2::Digest; let mut hasher = sha2::Sha256::new(); hasher.update(parent.as_slice()); hasher.update(&attributes.timestamp.to_be_bytes()[..]); diff --git a/crates/prune/prune/src/segments/user/account_history.rs b/crates/prune/prune/src/segments/user/account_history.rs index 440617f8ac7..32c9fa3c956 100644 --- a/crates/prune/prune/src/segments/user/account_history.rs +++ b/crates/prune/prune/src/segments/user/account_history.rs @@ -350,10 +350,17 @@ impl AccountHistory { mod tests { use super::ACCOUNT_HISTORY_TABLES_TO_PRUNE; use crate::segments::{AccountHistory, PruneInput, PruneLimiter, Segment, SegmentOutput}; - use alloy_primitives::{BlockNumber, B256}; + use alloy_primitives::{Address, BlockNumber, B256, U256}; use assert_matches::assert_matches; - use reth_db_api::{models::StorageSettings, tables, BlockNumberList}; - use reth_provider::{DBProvider, DatabaseProviderFactory, PruneCheckpointReader}; + use reth_db_api::{ + models::{ShardedKey, StorageSettings}, + tables, BlockNumberList, + }; + use reth_primitives_traits::Account; + use reth_provider::{ + DBProvider, DatabaseProviderFactory, PruneCheckpointReader, RocksDBProviderFactory, + StaticFileProviderFactory, + }; use reth_prune_types::{ PruneCheckpoint, PruneInterruptReason, PruneMode, PruneProgress, PruneSegment, }; @@ -361,6 +368,7 @@ mod tests { use reth_storage_api::StorageSettingsCache; use reth_testing_utils::generators::{ self, random_block_range, random_changeset_range, random_eoa_accounts, BlockRangeParams, + ChangeSet, }; use std::{collections::BTreeMap, ops::AddAssign}; @@ -526,9 +534,6 @@ mod tests { #[test] fn prune_rocksdb_path() { - use reth_db_api::models::ShardedKey; - use reth_provider::{RocksDBProviderFactory, StaticFileProviderFactory}; - let db = TestStageDB::default(); let mut rng = generators::rng(); @@ -631,10 +636,6 @@ mod tests { #[test] #[allow(clippy::clone_on_copy)] fn prune_partial_progress_mid_block() { - use alloy_primitives::{Address, U256}; - use reth_primitives_traits::Account; - use reth_testing_utils::generators::ChangeSet; - let db = TestStageDB::default(); let mut rng = generators::rng(); @@ -789,9 +790,6 @@ mod tests { /// every later run reread it and never advance. #[test] fn dense_block_advances_rocksdb_checkpoint() { - use reth_db_api::models::ShardedKey; - use reth_provider::RocksDBProviderFactory; - let db = TestStageDB::default(); let mut rng = generators::rng(); diff --git a/crates/prune/prune/src/segments/user/storage_history.rs b/crates/prune/prune/src/segments/user/storage_history.rs index d2aa4052f3a..7dc8915b685 100644 --- a/crates/prune/prune/src/segments/user/storage_history.rs +++ b/crates/prune/prune/src/segments/user/storage_history.rs @@ -357,10 +357,16 @@ impl StorageHistory { mod tests { use super::STORAGE_HISTORY_TABLES_TO_PRUNE; use crate::segments::{PruneInput, PruneLimiter, Segment, SegmentOutput, StorageHistory}; - use alloy_primitives::{BlockNumber, B256}; + use alloy_primitives::{Address, BlockNumber, B256, U256}; use assert_matches::assert_matches; - use reth_db_api::{models::StorageSettings, tables, BlockNumberList}; - use reth_provider::{DBProvider, DatabaseProviderFactory, PruneCheckpointReader}; + use reth_db_api::{ + models::{storage_sharded_key::StorageShardedKey, StorageSettings}, + tables, BlockNumberList, + }; + use reth_primitives_traits::{Account, StorageEntry}; + use reth_provider::{ + DBProvider, DatabaseProviderFactory, PruneCheckpointReader, RocksDBProviderFactory, + }; use reth_prune_types::{ PruneCheckpoint, PruneInterruptReason, PruneMode, PruneProgress, PruneSegment, }; @@ -368,6 +374,7 @@ mod tests { use reth_storage_api::StorageSettingsCache; use reth_testing_utils::generators::{ self, random_block_range, random_changeset_range, random_eoa_accounts, BlockRangeParams, + ChangeSet, }; use std::{collections::BTreeMap, ops::AddAssign}; @@ -541,10 +548,6 @@ mod tests { #[test] #[allow(clippy::clone_on_copy)] fn prune_partial_progress_mid_block() { - use alloy_primitives::{Address, U256}; - use reth_primitives_traits::Account; - use reth_testing_utils::generators::ChangeSet; - let db = TestStageDB::default(); let mut rng = generators::rng(); @@ -689,8 +692,6 @@ mod tests { #[test] fn prune_rocksdb() { - use reth_db_api::models::storage_sharded_key::StorageShardedKey; - use reth_provider::RocksDBProviderFactory; use reth_storage_api::StorageSettings; let db = TestStageDB::default(); @@ -800,10 +801,6 @@ mod tests { #[test] #[allow(clippy::clone_on_copy)] fn dense_block_advances_rocksdb_checkpoint() { - use alloy_primitives::U256; - use reth_db_api::models::storage_sharded_key::StorageShardedKey; - use reth_primitives_traits::StorageEntry; - use reth_provider::RocksDBProviderFactory; use reth_storage_api::StorageSettings; let db = TestStageDB::default(); diff --git a/crates/prune/prune/src/segments/user/transaction_lookup.rs b/crates/prune/prune/src/segments/user/transaction_lookup.rs index 1aed8438302..ec67b85e334 100644 --- a/crates/prune/prune/src/segments/user/transaction_lookup.rs +++ b/crates/prune/prune/src/segments/user/transaction_lookup.rs @@ -309,12 +309,17 @@ mod tests { FoldWhile::{Continue, Done}, Itertools, }; - use reth_db_api::tables; - use reth_provider::{DBProvider, DatabaseProviderFactory, PruneCheckpointReader}; + use reth_db_api::{models::StorageSettings, tables}; + use reth_provider::{ + DBProvider, DatabaseProviderFactory, PruneCheckpointReader, RocksDBProviderFactory, + StaticFileProviderFactory, StaticFileWriter, + }; use reth_prune_types::{ PruneCheckpoint, PruneInterruptReason, PruneMode, PruneProgress, PruneSegment, }; use reth_stages::test_utils::{StorageKind, TestStageDB}; + use reth_static_file_types::{StaticFileSegment, DEFAULT_BLOCKS_PER_STATIC_FILE}; + use reth_storage_api::StorageSettingsCache; use reth_testing_utils::generators::{self, random_block_range, BlockRangeParams}; use std::ops::Sub; @@ -363,9 +368,6 @@ mod tests { #[test] fn prune_transaction_zero_when_earlier_blocks_are_empty() { - use reth_provider::{StaticFileProviderFactory, StaticFileWriter}; - use reth_static_file_types::{StaticFileSegment, DEFAULT_BLOCKS_PER_STATIC_FILE}; - let db = TestStageDB::default(); let mut rng = generators::rng(); let first_retained_block = DEFAULT_BLOCKS_PER_STATIC_FILE; @@ -558,10 +560,6 @@ mod tests { #[test] fn prune_rocksdb() { - use reth_db_api::models::StorageSettings; - use reth_provider::RocksDBProviderFactory; - use reth_storage_api::StorageSettingsCache; - let db = TestStageDB::default(); let mut rng = generators::rng(); @@ -658,10 +656,6 @@ mod tests { /// 3. The checkpoint should NOT advance to the next start position #[test] fn prune_rocksdb_zero_deleted_checkpoint() { - use reth_db_api::models::StorageSettings; - use reth_provider::RocksDBProviderFactory; - use reth_storage_api::StorageSettingsCache; - let db = TestStageDB::default(); let mut rng = generators::rng(); diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index ee8d353a1a4..813150ea990 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -1979,11 +1979,7 @@ impl TransportRpcModules { /// it's the caller responsibility to remove both `subscribe` and `unsubscribe` methods for /// subscriptions. pub fn remove_http_method(&mut self, method_name: &'static str) -> bool { - if let Some(http_module) = &mut self.http { - http_module.remove_method(method_name).is_some() - } else { - false - } + self.http.as_mut().is_some_and(|module| module.remove_method(method_name).is_some()) } /// Removes the given methods from the configured http methods. @@ -2001,11 +1997,7 @@ impl TransportRpcModules { /// it's the caller responsibility to remove both `subscribe` and `unsubscribe` methods for /// subscriptions. pub fn remove_ws_method(&mut self, method_name: &'static str) -> bool { - if let Some(ws_module) = &mut self.ws { - ws_module.remove_method(method_name).is_some() - } else { - false - } + self.ws.as_mut().is_some_and(|module| module.remove_method(method_name).is_some()) } /// Removes the given methods from the configured ws methods. @@ -2023,11 +2015,7 @@ impl TransportRpcModules { /// it's the caller responsibility to remove both `subscribe` and `unsubscribe` methods for /// subscriptions. pub fn remove_ipc_method(&mut self, method_name: &'static str) -> bool { - if let Some(ipc_module) = &mut self.ipc { - ipc_module.remove_method(method_name).is_some() - } else { - false - } + self.ipc.as_mut().is_some_and(|module| module.remove_method(method_name).is_some()) } /// Removes the given methods from the configured ipc methods. diff --git a/crates/rpc/rpc-builder/tests/it/http.rs b/crates/rpc/rpc-builder/tests/it/http.rs index 5bcedef37d6..4c1aa08dfb4 100644 --- a/crates/rpc/rpc-builder/tests/it/http.rs +++ b/crates/rpc/rpc-builder/tests/it/http.rs @@ -556,14 +556,13 @@ where .unwrap() .is_none()); TraceApiClient::::trace_block(client, block_id).await.unwrap_err(); - assert!(TraceApiClient::::replay_block_transactions( + TraceApiClient::::replay_block_transactions( client, block_id, HashSet::default(), ) .await - .unwrap() - .is_none()); + .unwrap_err(); TraceApiClient::::trace_filter(client, trace_filter).await.unwrap(); } diff --git a/crates/rpc/rpc-e2e-tests/src/rpc_compat.rs b/crates/rpc/rpc-e2e-tests/src/rpc_compat.rs index c39987cba5d..f37d93d9fc4 100644 --- a/crates/rpc/rpc-e2e-tests/src/rpc_compat.rs +++ b/crates/rpc/rpc-e2e-tests/src/rpc_compat.rs @@ -2,7 +2,7 @@ use eyre::{eyre, Result}; use futures_util::future::BoxFuture; -use jsonrpsee::core::client::ClientT; +use jsonrpsee::core::{client::ClientT, params::ArrayParams}; use reth_e2e_test_utils::testsuite::{ actions::{expect_fcu_valid, Action}, BlockInfo, Environment, @@ -172,7 +172,6 @@ impl RunRpcCompatTests { // Make the RPC request using jsonrpsee // We need to handle the case where the RPC might return an error - use jsonrpsee::core::params::ArrayParams; let response_result: Result = match params { Value::Array(ref arr) => { diff --git a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs index a8c9df586f5..c8d24358689 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs @@ -28,6 +28,7 @@ use reth_storage_api::{ noop::NoopProvider, BlockReader, BlockReaderIdExt, ProviderHeader, ProviderTx, StateProvider, StateProviderBox, StateProviderFactory, }; +use reth_tasks::cancel::is_cancelled; use reth_transaction_pool::{ error::InvalidPoolTransactionError, BestTransactions, BestTransactionsAttributes, PoolTransaction, TransactionPool, @@ -326,6 +327,12 @@ pub trait LoadPendingBlock: .without_updates(); while let Some(pool_tx) = best_txs.next() { + // nobody waits for the block once the request is dropped, and the cache lock was + // released with it + if is_cancelled() { + return Err(Self::Error::from_eth_err(EthApiError::InternalEthError)) + } + // ensure we still have capacity for this transaction let exceeds_gas_limit = if is_amsterdam { let regular_available_gas = diff --git a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs index 88d710e6b59..f191a7fd9ea 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs @@ -163,8 +163,7 @@ pub trait EthTransactions: LoadTransaction { receipt, this.converter(), ) - .transpose() - .map_err(Self::Error::from)? + .transpose()? { return Ok(receipt); } diff --git a/crates/rpc/rpc-eth-types/src/error/mod.rs b/crates/rpc/rpc-eth-types/src/error/mod.rs index 5979c27f252..9afcea8ee18 100644 --- a/crates/rpc/rpc-eth-types/src/error/mod.rs +++ b/crates/rpc/rpc-eth-types/src/error/mod.rs @@ -10,7 +10,7 @@ pub use api::{AsEthApiError, FromEthApiError, FromEvmError, IntoEthApiError}; use core::time::Duration; use evm2::{interpreter::InstrStop, registry::HandlerError}; use evm2_inspectors::tracing::{DebugInspectorError, MuxError}; -use reth_errors::{BlockExecutionError, BlockValidationError, RethError}; +use reth_errors::{BlockExecutionError, BlockValidationError, ProviderError, RethError}; use reth_primitives_traits::transaction::{error::InvalidTransactionError, signed::RecoveryError}; use reth_rpc_convert::{CallFeesError, EthTxEnvError, TransactionConversionError}; use reth_rpc_server_types::result::{ @@ -568,7 +568,6 @@ impl From for EthApiError { impl From for EthApiError { fn from(error: reth_errors::ProviderError) -> Self { - use reth_errors::ProviderError; match error { ProviderError::HeaderNotFound(hash) => Self::HeaderNotFound(hash.into()), ProviderError::BlockHashNotFound(hash) | ProviderError::UnknownBlockHash(hash) => { @@ -827,12 +826,7 @@ impl From for jsonrpsee_types::error::ErrorObject<'s fn from(err: RpcInvalidTransactionError) -> Self { match err { RpcInvalidTransactionError::Revert(revert) => { - // include out data if some - rpc_err( - revert.error_code(), - revert.to_string(), - revert.output.as_ref().map(|out| out.as_ref()), - ) + rpc_err(revert.error_code(), revert.to_string(), Some(revert.output.as_ref())) } RpcInvalidTransactionError::Other(err) => err.to_rpc_error(), err => rpc_err(err.error_code(), err.to_string(), None), @@ -842,7 +836,6 @@ impl From for jsonrpsee_types::error::ErrorObject<'s impl From for RpcInvalidTransactionError { fn from(err: InvalidTransactionError) -> Self { - use InvalidTransactionError; // This conversion is used to convert any transaction errors that could occur inside the // txpool (e.g. `eth_sendRawTransaction`) to their corresponding RPC match err { @@ -915,9 +908,7 @@ impl From for RpcInvalidTransactionError { #[derive(Debug, Clone, thiserror::Error)] pub struct RevertError { /// The transaction output data - /// - /// Note: this is `None` if output was empty - output: Option, + output: Bytes, } // === impl RevertError == @@ -926,12 +917,8 @@ impl RevertError { /// Wraps the output bytes /// /// Note: this is intended to wrap an EVM revert output - pub fn new(output: Bytes) -> Self { - if output.is_empty() { - Self { output: None } - } else { - Self { output: Some(output) } - } + pub const fn new(output: Bytes) -> Self { + Self { output } } /// Returns error code to return for this error. @@ -943,7 +930,9 @@ impl RevertError { impl std::fmt::Display for RevertError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str("execution reverted")?; - if let Some(reason) = self.output.as_ref().and_then(|out| RevertReason::decode(out)) { + if !self.output.is_empty() && + let Some(reason) = RevertReason::decode(&self.output) + { let error = reason.to_string(); let mut error = error.as_str(); if reason.is_revert() { @@ -1138,6 +1127,7 @@ mod tests { use super::*; use alloy_primitives::b256; use alloy_sol_types::{Revert, SolError}; + use reth_errors::ProviderError; #[test] fn timed_out_error() { @@ -1217,8 +1207,6 @@ mod tests { #[test] fn pruned_state_errors_use_history_unavailable_code() { - use reth_errors::ProviderError; - let err = EthApiError::from(ProviderError::InsufficientChangesets { requested: 1, available: 43..=48, @@ -1288,4 +1276,13 @@ mod tests { let msg = err.to_string(); assert_eq!(msg, "execution reverted: test_revert_reason"); } + + #[test] + fn empty_revert_err_has_empty_data() { + let err: jsonrpsee_types::error::ErrorObject<'static> = + RpcInvalidTransactionError::Revert(RevertError::new(Bytes::new())).into(); + assert_eq!(err.code(), 3); + assert_eq!(err.message(), "execution reverted"); + assert_eq!(err.data().unwrap().get(), "\"0x\""); + } } diff --git a/crates/rpc/rpc-eth-types/src/simulate.rs b/crates/rpc/rpc-eth-types/src/simulate.rs index 149b22a11aa..d5dd4b559d3 100644 --- a/crates/rpc/rpc-eth-types/src/simulate.rs +++ b/crates/rpc/rpc-eth-types/src/simulate.rs @@ -486,12 +486,10 @@ where // gas price. let tx_type = tx.as_ref().output_tx_type(); - let from = if let Some(from) = tx.as_ref().from() { - from - } else { + let from = tx.as_ref().from().unwrap_or_else(|| { tx.as_mut().set_from(Address::ZERO); Address::ZERO - }; + }); let state_nonce = account(&from)?.map(|acc| acc.nonce).unwrap_or_default(); if tx.as_ref().nonce().is_none() { diff --git a/crates/rpc/rpc/src/eth/bundle.rs b/crates/rpc/rpc/src/eth/bundle.rs index dff5a421f28..7aa2e1f4b35 100644 --- a/crates/rpc/rpc/src/eth/bundle.rs +++ b/crates/rpc/rpc/src/eth/bundle.rs @@ -16,7 +16,7 @@ use reth_rpc_eth_types::{ EthApiError, PendingBlockEnv, PendingBlockEnvOrigin, RpcInvalidTransactionError, }; use reth_storage_api::BlockReaderIdExt; -use reth_tasks::pool::BlockingTaskGuard; +use reth_tasks::{cancel::is_cancelled, pool::BlockingTaskGuard}; use reth_transaction_pool::{ EthBlobTransactionSidecar, EthPoolTransaction, PoolPooledTx, PoolTransaction, TransactionPool, }; @@ -146,7 +146,8 @@ where } // Apply gas limit: default to call gas limit unless user requests a smaller limit - evm_env.block_env_mut().gas_limit = U256::from(gas_limit.unwrap_or(call_gas_limit)); + let block_gas_limit = gas_limit.unwrap_or(call_gas_limit); + evm_env.block_env_mut().gas_limit = U256::from(block_gas_limit); // The bundle is simulated on top of the state block, so default to the next block's base // fee. @@ -165,8 +166,15 @@ where // use the block number of the request evm_env.block_env_mut().number = U256::from(block_number); + let permit = self + .eth_api() + .acquire_owned_blocking_io() + .await + .map_err(|_| EthApiError::InternalEthError)?; + self.eth_api() .spawn_with_state_at_block(at, move |eth_api, mut db| { + let _permit = permit; let coinbase = evm_env.block_env().beneficiary; let basefee = evm_env.block_base_fee(); @@ -175,15 +183,39 @@ where .map_err(Eth::Error::from_eth_err)? .map(|acc| acc.balance) .unwrap_or_default(); + let separate_block_gas = evm_env.version().feature(evm2::EvmFeatures::EIP8037); + let tx_gas_limit_cap = evm_env.version().tx_gas_limit_cap; let mut evm = eth_api.evm_config().evm_with_env(&mut db, evm_env); let mut coinbase_balance_before_tx = initial_coinbase; let mut coinbase_balance_after_tx = initial_coinbase; let mut total_gas_used = 0u64; + let mut regular_gas_used = 0u64; + let mut state_gas_used = 0u64; let mut total_gas_fees = U256::ZERO; let mut hasher = Keccak256::new(); let mut results = Vec::with_capacity(transactions.len()); for tx in transactions { + if is_cancelled() { + return Err(EthApiError::InternalEthError.into()) + } + + // Admit each transaction against the gas left in the block, like the block + // executor does: EIP-8037 keeps separate regular and state gas pools. + let regular_limit = tx.gas_limit().min(tx_gas_limit_cap); + let exceeds_block_gas = if separate_block_gas { + regular_limit > block_gas_limit.saturating_sub(regular_gas_used) || + tx.gas_limit() > block_gas_limit.saturating_sub(state_gas_used) + } else { + regular_limit > block_gas_limit.saturating_sub(total_gas_used) + }; + if exceeds_block_gas { + return Err(EthApiError::InvalidParams( + EthBundleError::BundleGasLimitExceeded(block_gas_limit).to_string(), + ) + .into()) + } + let signer = tx.signer(); let tx = { let mut tx = ::Transaction::from_pooled(tx); @@ -215,6 +247,8 @@ where let gas_used = result.tx_gas_used(); total_gas_used += gas_used; + regular_gas_used += result.execution_gas_spent(); + state_gas_used += result.state_gas_spent(); let gas_fees = U256::from(gas_used) * U256::from(gas_price); total_gas_fees += gas_fees; @@ -322,6 +356,9 @@ pub enum EthBundleError { /// Thrown when the blob gas usage of the blob transactions in a bundle exceed the maximum. #[error("blob gas usage exceeds the limit of {0} gas per block.")] Eip4844BlobGasExceeded(u64), + /// Thrown when the gas limits of the bundle's transactions exceed the block gas limit. + #[error("bundle gas exceeds the block gas limit of {0}")] + BundleGasLimitExceeded(u64), } #[cfg(test)] @@ -342,6 +379,7 @@ mod tests { use reth_provider::{ providers::BlockchainProvider, test_utils::create_test_provider_factory_with_chain_spec, }; + use reth_rpc_eth_api::helpers::SpawnBlocking; use reth_storage_overlay::OverlayManager; use reth_testing_utils::generators::{self, generate_key, sign_tx_with_key_pair}; use reth_transaction_pool::test_utils::testing_pool; @@ -457,6 +495,149 @@ mod tests { } } + /// The bundle's transactions share the block gas limit, and the simulation waits for a + /// blocking IO permit. + #[tokio::test(flavor = "multi_thread")] + async fn call_bundle_shares_gas_limit_and_takes_blocking_io_permit() { + let mut rng = generators::rng(); + let key = generate_key(&mut rng); + let txs: Vec<_> = (0..2) + .map(|nonce| { + sign_tx_with_key_pair( + key, + reth_ethereum_primitives::Transaction::Eip1559(TxEip1559 { + chain_id: 1, + nonce, + gas_limit: 21_000, + max_fee_per_gas: INITIAL_BASE_FEE as u128, + to: TxKind::Call(Address::random()), + ..Default::default() + }), + ) + }) + .collect(); + let sender = txs[0].recover_signer().unwrap(); + + let genesis = Genesis::default() + .with_gas_limit(30_000_000) + .with_base_fee(Some(INITIAL_BASE_FEE as u128)) + .extend_accounts([( + sender, + GenesisAccount::default().with_balance(U256::from(10u128.pow(18))), + )]); + let chain_spec = + Arc::new(ChainSpecBuilder::mainnet().cancun_activated().genesis(genesis).build()); + let factory = create_test_provider_factory_with_chain_spec(chain_spec); + init_genesis(&factory).unwrap(); + let provider = BlockchainProvider::new(factory).unwrap(); + + let eth_api = EthApiBuilder::new( + provider.clone(), + testing_pool(), + NoopNetwork::default(), + EthEvmConfig::new(provider.chain_spec()), + ) + .max_blocking_io_requests(1) + .build(); + let api = EthBundle::new(eth_api.clone(), BlockingTaskGuard::new(4)); + + let bundle = EthCallBundle { + txs: txs.iter().map(|tx| tx.encoded_2718().into()).collect(), + block_number: 1, + state_block_number: BlockNumberOrTag::Number(0), + ..Default::default() + }; + + let error = api + .call_bundle(EthCallBundle { gas_limit: Some(41_999), ..bundle.clone() }) + .await + .unwrap_err(); + assert_eq!( + error.to_string(), + EthApiError::InvalidParams(EthBundleError::BundleGasLimitExceeded(41_999).to_string()) + .to_string() + ); + + let bundle = EthCallBundle { gas_limit: Some(42_000), ..bundle }; + let permit = eth_api.acquire_owned_blocking_io().await.unwrap(); + assert!(tokio::time::timeout( + std::time::Duration::from_millis(200), + api.call_bundle(bundle.clone()) + ) + .await + .is_err()); + + drop(permit); + let response = api.call_bundle(bundle).await.unwrap(); + assert_eq!(response.total_gas_used, 42_000); + } + + /// Under EIP-8037 the bundle admits transactions against separate regular and state gas + /// pools, so a bundle that fits a block is accepted even when its receipt gas does not fit + /// one pool. + #[tokio::test(flavor = "multi_thread")] + async fn call_bundle_uses_separate_gas_pools_after_amsterdam() { + const TX_GAS_LIMIT: u64 = 1_000_000; + + let mut rng = generators::rng(); + let key = generate_key(&mut rng); + // Value transfers to new accounts spend both regular and state gas. + let txs: Vec<_> = (0..2) + .map(|nonce| { + sign_tx_with_key_pair( + key, + reth_ethereum_primitives::Transaction::Eip1559(TxEip1559 { + chain_id: 1, + nonce, + gas_limit: TX_GAS_LIMIT, + max_fee_per_gas: INITIAL_BASE_FEE as u128, + to: TxKind::Call(Address::random()), + value: U256::from(1), + ..Default::default() + }), + ) + }) + .collect(); + let sender = txs[0].recover_signer().unwrap(); + + let genesis = Genesis::default() + .with_gas_limit(30_000_000) + .with_base_fee(Some(INITIAL_BASE_FEE as u128)) + .extend_accounts([( + sender, + GenesisAccount::default().with_balance(U256::from(10u128.pow(18))), + )]); + let chain_spec = + Arc::new(ChainSpecBuilder::mainnet().amsterdam_activated().genesis(genesis).build()); + let factory = create_test_provider_factory_with_chain_spec(chain_spec); + init_genesis(&factory).unwrap(); + let provider = BlockchainProvider::new(factory).unwrap(); + + let eth_api = EthApiBuilder::new( + provider.clone(), + testing_pool(), + NoopNetwork::default(), + EthEvmConfig::new(provider.chain_spec()), + ) + .build(); + let api = EthBundle::new(eth_api, BlockingTaskGuard::new(4)); + + let bundle = EthCallBundle { + txs: txs.iter().map(|tx| tx.encoded_2718().into()).collect(), + block_number: 1, + state_block_number: BlockNumberOrTag::Number(0), + ..Default::default() + }; + let first_gas_used = api.call_bundle(bundle.clone()).await.unwrap().results[0].gas_used; + + // The second transaction fits both pools, but not one pool holding the first + // transaction's regular and state gas together. + let gas_limit = TX_GAS_LIMIT + first_gas_used - 1; + let response = + api.call_bundle(EthCallBundle { gas_limit: Some(gas_limit), ..bundle }).await.unwrap(); + assert_eq!(response.results.len(), 2); + } + /// A pre-London state has no base fee check in the EVM, so the bundle must reject a fee cap /// below the supplied base fee instead of panicking. #[tokio::test(flavor = "multi_thread")] diff --git a/crates/rpc/rpc/src/eth/core.rs b/crates/rpc/rpc/src/eth/core.rs index ca76ca8a80e..b4bdd332604 100644 --- a/crates/rpc/rpc/src/eth/core.rs +++ b/crates/rpc/rpc/src/eth/core.rs @@ -547,6 +547,7 @@ mod tests { state::{AccountOverride, EvmOverrides, StateOverride}, BlockOverrides, Bundle, TransactionRequest, }; + use alloy_rpc_types_trace::{parity::TraceType, tracerequest::TraceCallRequest}; use evm2::{evm::CacheDB, AccountInfo}; use jsonrpsee_types::error::INVALID_PARAMS_CODE; use rand::Rng; @@ -557,7 +558,9 @@ mod tests { use reth_evm::{ConfigureEvm, EvmEnvFor, TxEnvFor}; use reth_evm_ethereum::EthEvmConfig; use reth_network_api::noop::NoopNetwork; - use reth_primitives_traits::crypto::secp256k1::sign_message; + use reth_primitives_traits::{ + crypto::secp256k1::sign_message, SealedHeader, SignerRecoverable, + }; use reth_provider::{ providers::BlockchainProvider, test_utils::{ @@ -568,17 +571,17 @@ mod tests { }; use reth_rpc_convert::{EthTxEnvError, TryIntoTxEnv}; use reth_rpc_eth_api::{ - helpers::{Call, EthBlocks, EthCall, SpawnBlocking}, + helpers::{Call, EthBlocks, EthCall, EthTransactions, LoadPendingBlock, SpawnBlocking}, node::RpcNodeCoreAdapter, EthApiServer, }; - use reth_rpc_eth_types::{receipt::EthReceiptConverter, RpcInvalidTransactionError}; + use reth_rpc_eth_types::{receipt::EthReceiptConverter, EthConfig, RpcInvalidTransactionError}; use reth_storage_api::{ BalProvider, BlockReaderIdExt, NodePrimitivesProvider, StateProviderFactory, }; - use reth_tasks::cancel::is_cancelled; + use reth_tasks::cancel::{is_cancelled, CancelOnDrop}; use reth_testing_utils::generators; - use reth_transaction_pool::test_utils::{testing_pool, TestPool}; + use reth_transaction_pool::test_utils::{testing_pool, MockTransaction, TestPool}; use std::time::{Duration, Instant}; type FakeEthApi

= EthApi< @@ -649,10 +652,28 @@ mod tests { assert!(cancelled_rx.await.unwrap()); } + #[tokio::test(flavor = "multi_thread")] + async fn pending_block_build_stops_when_cancelled() { + let provider = MockEthProvider::default(); + let parent = + Header { gas_limit: 30_000_000, base_fee_per_gas: Some(1), ..Default::default() }; + let parent_hash = B256::random(); + provider.add_header(parent_hash, parent.clone()); + let api = build_test_eth_api(provider); + api.pool().add_external_transaction(MockTransaction::eip1559()).await.unwrap(); + + // dropping the request's handle cancels the build running in its scope + let request = CancelOnDrop::default(); + let scope = request.clone(); + drop(request); + + let parent = SealedHeader::new(parent, parent_hash); + let err = scope.scope(|| api.build_block(&parent)).unwrap_err(); + assert!(matches!(err, EthApiError::InternalEthError)); + } + #[tokio::test] async fn test_transaction_by_sender_and_nonce_without_sender_transaction_returns_none() { - use reth_rpc_eth_api::helpers::EthTransactions; - let provider = MockEthProvider::default(); let sender = Address::random(); provider.add_account(sender, ExtendedAccount::new(1, U256::ZERO)); @@ -677,7 +698,6 @@ mod tests { #[tokio::test] async fn test_transaction_by_sender_and_nonce_missing_block_returns_error() { - use reth_rpc_eth_api::helpers::EthTransactions; use reth_rpc_eth_types::EthApiError; let provider = MockEthProvider::default(); @@ -695,9 +715,6 @@ mod tests { #[tokio::test] async fn test_transaction_by_sender_and_nonce_returns_matching_transaction() { - use reth_primitives_traits::SignerRecoverable; - use reth_rpc_eth_api::helpers::EthTransactions; - let provider = MockEthProvider::default(); let tx = TransactionSigned::new_unhashed( reth_ethereum_primitives::Transaction::Legacy(Default::default()), @@ -1135,8 +1152,6 @@ mod tests { #[tokio::test] async fn call_allowance_respects_rpc_gas_cap() { - use alloy_rpc_types_trace::{parity::TraceType, tracerequest::TraceCallRequest}; - use reth_rpc_eth_types::EthConfig; use reth_tasks::pool::BlockingTaskGuard; let sender = Address::repeat_byte(0x11); diff --git a/crates/rpc/rpc/src/eth/filter.rs b/crates/rpc/rpc/src/eth/filter.rs index 57919d47164..2d4e0ece1ce 100644 --- a/crates/rpc/rpc/src/eth/filter.rs +++ b/crates/rpc/rpc/src/eth/filter.rs @@ -1407,16 +1407,18 @@ impl< mod tests { use super::*; use crate::{eth::EthApi, EthApiBuilder}; + use alloy_consensus::TxLegacy; use alloy_network::Ethereum; use alloy_primitives::{Bloom, FixedBytes}; use rand::Rng; use reth_chainspec::{ChainSpec, ChainSpecProvider}; + use reth_db_api::models::StoredBlockBodyIndices; use reth_ethereum_primitives::TxType; use reth_evm_ethereum::EthEvmConfig; use reth_network_api::noop::NoopNetwork; use reth_provider::test_utils::MockEthProvider; use reth_rpc_convert::RpcConverter; - use reth_rpc_eth_api::node::RpcNodeCoreAdapter; + use reth_rpc_eth_api::{helpers::SpawnBlocking, node::RpcNodeCoreAdapter}; use reth_rpc_eth_types::receipt::EthReceiptConverter; use reth_tasks::Runtime; use reth_testing_utils::generators; @@ -1923,8 +1925,6 @@ mod tests { async fn test_log_limit_retry_range_excludes_overflow_block() { let provider = MockEthProvider::default(); - use alloy_consensus::TxLegacy; - use reth_db_api::models::StoredBlockBodyIndices; use reth_ethereum_primitives::{TransactionSigned, TxType}; let tx_inner = TxLegacy { @@ -2009,7 +2009,6 @@ mod tests { let mut prev_hash = alloy_primitives::B256::default(); // Create a transaction for blocks that will have receipts - use alloy_consensus::TxLegacy; use reth_ethereum_primitives::{TransactionSigned, TxType}; let tx_inner = TxLegacy { @@ -2070,7 +2069,6 @@ mod tests { provider.add_receipts(103, vec![]); // Add block body indices for each block so receipts can be fetched - use reth_db_api::models::StoredBlockBodyIndices; provider .add_block_body_indices(100, StoredBlockBodyIndices { first_tx_num: 0, tx_count: 1 }); provider @@ -2107,8 +2105,6 @@ mod tests { #[tokio::test] async fn test_range_scan_waits_for_blocking_io_permit() { - use reth_rpc_eth_api::helpers::SpawnBlocking; - let provider = MockEthProvider::default(); let header = alloy_consensus::Header::default(); provider.add_header(header.hash_slow(), header); @@ -2139,9 +2135,7 @@ mod tests { #[tokio::test] async fn test_range_scan_across_windows() { - use alloy_consensus::TxLegacy; use alloy_primitives::{Address, Bloom, Bytes, Log, LogData, Signature}; - use reth_db_api::models::StoredBlockBodyIndices; use reth_ethereum_primitives::{Block, BlockBody, Receipt, TransactionSigned}; let provider = MockEthProvider::default(); diff --git a/crates/rpc/rpc/src/eth/helpers/sync_listener.rs b/crates/rpc/rpc/src/eth/helpers/sync_listener.rs index 16e643d726a..86c33a57858 100644 --- a/crates/rpc/rpc/src/eth/helpers/sync_listener.rs +++ b/crates/rpc/rpc/src/eth/helpers/sync_listener.rs @@ -68,6 +68,8 @@ mod tests { Arc, }, }; + use tokio::sync::mpsc::unbounded_channel; + use tokio_stream::wrappers::UnboundedReceiverStream; #[derive(Clone)] struct TestNetwork { @@ -122,9 +124,6 @@ mod tests { #[tokio::test] async fn resolves_when_syncing_stops() { - use tokio::sync::mpsc::unbounded_channel; - use tokio_stream::wrappers::UnboundedReceiverStream; - let syncing = Arc::new(AtomicBool::new(true)); let network = TestNetwork { syncing: syncing.clone() }; let (tx, rx) = unbounded_channel(); diff --git a/crates/rpc/rpc/src/eth/sim_bundle.rs b/crates/rpc/rpc/src/eth/sim_bundle.rs index 3110498c9e3..b23ae394e5b 100644 --- a/crates/rpc/rpc/src/eth/sim_bundle.rs +++ b/crates/rpc/rpc/src/eth/sim_bundle.rs @@ -415,8 +415,6 @@ where if logs { Self::build_bundle_logs(&request, &flat_logs)? } else { vec![] }; // After processing all transactions, process refunds - // Store the original refundable value to calculate all payouts correctly - let original_refundable_value = refundable_value; for item in &flattened_bundle { if let Some(refund_percent) = item.refund_percent { let refund_configs = item.refund_configs.clone().unwrap_or_else(|| { @@ -431,11 +429,9 @@ where // Add gas used for payout transactions total_gas_used += SBUNDLE_PAYOUT_MAX_COST * refund_configs.len() as u64; - // Calculate allocated refundable value (payout value) based on ORIGINAL - // refundable value. This ensures all refund_percent values are - // calculated from the same base. - let payout_value = original_refundable_value * U256::from(refund_percent) / - U256::from(100); + // Calculate allocated refundable value (payout value) + let payout_value = + refundable_value * U256::from(refund_percent) / U256::from(100); if payout_tx_fee > payout_value { return Err(EthApiError::InvalidParams( @@ -450,13 +446,6 @@ where EthSimBundleError::NegativeProfit.to_string(), ), )?; - - // Adjust refundable value - refundable_value = refundable_value.checked_sub(payout_value).ok_or( - EthApiError::InvalidParams( - EthSimBundleError::NegativeProfit.to_string(), - ), - )?; } } @@ -573,27 +562,28 @@ pub enum EthSimBundleError { #[cfg(test)] mod tests { use super::*; - use alloy_primitives::Bytes; + use crate::EthApiBuilder; + use alloy_consensus::{transaction::SignerRecoverable, TxEip1559}; + use alloy_eips::eip2718::Encodable2718; + use alloy_genesis::{Genesis, GenesisAccount}; + use alloy_primitives::{Address, Bytes, TxKind, B256}; + use alloy_rpc_types_eth::BlockOverrides; use alloy_rpc_types_mev::{Inclusion, ProtocolVersion}; + use reth_chainspec::{ChainSpecBuilder, ChainSpecProvider}; + use reth_db_common::init::init_genesis; + use reth_evm_ethereum::EthEvmConfig; + use reth_network_api::noop::NoopNetwork; + use reth_provider::{ + providers::BlockchainProvider, test_utils::create_test_provider_factory_with_chain_spec, + }; + use reth_testing_utils::{ + generators, + generators::{generate_key, sign_tx_with_key_pair}, + }; + use reth_transaction_pool::test_utils::testing_pool; #[tokio::test(flavor = "multi_thread")] async fn sim_bundle_uses_next_block_env_and_overrides() { - use crate::EthApiBuilder; - use alloy_consensus::{transaction::SignerRecoverable, TxEip1559}; - use alloy_eips::eip2718::Encodable2718; - use alloy_genesis::{Genesis, GenesisAccount}; - use alloy_primitives::{Address, TxKind, B256}; - use alloy_rpc_types_eth::BlockOverrides; - use reth_chainspec::{ChainSpecBuilder, ChainSpecProvider}; - use reth_db_common::init::init_genesis; - use reth_evm_ethereum::EthEvmConfig; - use reth_network_api::noop::NoopNetwork; - use reth_provider::{ - providers::BlockchainProvider, test_utils::create_test_provider_factory_with_chain_spec, - }; - use reth_testing_utils::generators::{self, generate_key, sign_tx_with_key_pair}; - use reth_transaction_pool::test_utils::testing_pool; - let contract = Address::repeat_byte(0xaa); let coinbase = Address::repeat_byte(0x11); let overridden_coinbase = Address::repeat_byte(0x22); diff --git a/crates/rpc/rpc/src/otterscan.rs b/crates/rpc/rpc/src/otterscan.rs index e081abfc532..ea08b7497a0 100644 --- a/crates/rpc/rpc/src/otterscan.rs +++ b/crates/rpc/rpc/src/otterscan.rs @@ -493,7 +493,9 @@ mod tests { use reth_network_api::noop::NoopNetwork; use reth_primitives_traits::Recovered; use reth_provider::test_utils::MockEthProvider; + use reth_rpc_eth_api::helpers::SpawnBlocking; use reth_transaction_pool::test_utils::testing_pool; + use std::time::Duration; #[test] fn block_transaction_pages_match_frontend_index_navigation() { @@ -703,9 +705,6 @@ mod tests { #[tokio::test] async fn trace_methods_wait_for_tracing_permit() { - use reth_rpc_eth_api::helpers::SpawnBlocking; - use std::time::Duration; - let api = OtterscanApi::new( crate::eth::EthApiBuilder::new( MockEthProvider::default(), diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index d117b3981df..0c92f9f5231 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -575,10 +575,14 @@ where block_id: BlockId, trace_types: HashSet, ) -> Result>, Eth::Error> { + let Some(block) = self.eth_api().recovered_block(block_id).await? else { + return Err(EthApiError::HeaderNotFound(block_id).into()); + }; + self.eth_api() .trace_block_with( block_id, - None, + Some(block), TracingInspectorConfig::from_parity_config(&trace_types), move |tx_info, ctx| { let full_trace = ctx @@ -908,16 +912,18 @@ pub struct BlockStorageAccess { mod tests { use super::*; use crate::EthApiBuilder; - use alloy_consensus::Header; + use alloy_consensus::{Header, TxLegacy}; use alloy_genesis::Genesis; - use alloy_primitives::bytes; + use alloy_primitives::{bytes, Signature, TxKind}; use alloy_rpc_types_eth::TransactionRequest; + use reth_chain_state::CanonStateNotification; use reth_chainspec::ChainSpecBuilder; use reth_db_common::init::init_genesis; - use reth_ethereum_primitives::{Block, BlockBody}; + use reth_ethereum_primitives::{Block, BlockBody, TransactionSigned}; use reth_evm_ethereum::EthEvmConfig; + use reth_execution_types::{Chain, ExecutionOutcome}; use reth_network_api::noop::NoopNetwork; - use reth_primitives_traits::Block as _; + use reth_primitives_traits::{Block as _, RecoveredBlock, SignerRecoverable}; use reth_provider::{ providers::BlockchainProvider, test_utils::{ @@ -925,6 +931,7 @@ mod tests { }, BlockWriter, StageCheckpointWriter, }; + use reth_rpc_eth_types::cache::cache_new_blocks_task; use reth_transaction_pool::test_utils::testing_pool; #[tokio::test] @@ -1008,19 +1015,6 @@ mod tests { #[tokio::test] async fn trace_get_selects_tree_paths() { - use crate::EthApiBuilder; - use alloy_consensus::{Header, TxLegacy}; - use alloy_primitives::{Signature, TxKind}; - use reth_chain_state::CanonStateNotification; - use reth_ethereum_primitives::{Block, BlockBody, TransactionSigned}; - use reth_evm_ethereum::EthEvmConfig; - use reth_execution_types::{Chain, ExecutionOutcome}; - use reth_network_api::noop::NoopNetwork; - use reth_primitives_traits::{RecoveredBlock, SignerRecoverable}; - use reth_provider::test_utils::{ExtendedAccount, MockEthProvider}; - use reth_rpc_eth_types::cache::cache_new_blocks_task; - use reth_transaction_pool::test_utils::testing_pool; - let provider = MockEthProvider::default(); // The root calls 0x43 (which calls 0x44), then calls 0x45. for (address, code) in [ @@ -1107,12 +1101,6 @@ mod tests { #[tokio::test] async fn replay_missing_transaction_returns_null() { - use crate::EthApiBuilder; - use reth_evm_ethereum::EthEvmConfig; - use reth_network_api::noop::NoopNetwork; - use reth_provider::test_utils::MockEthProvider; - use reth_transaction_pool::test_utils::testing_pool; - let provider = MockEthProvider::default(); let eth_api = EthApiBuilder::new( provider.clone(), @@ -1138,19 +1126,6 @@ mod tests { #[tokio::test] async fn replay_transaction_includes_transaction_hash() { - use crate::EthApiBuilder; - use alloy_consensus::{Header, TxLegacy}; - use alloy_primitives::{Signature, TxKind}; - use reth_chain_state::CanonStateNotification; - use reth_ethereum_primitives::{Block, BlockBody, TransactionSigned}; - use reth_evm_ethereum::EthEvmConfig; - use reth_execution_types::{Chain, ExecutionOutcome}; - use reth_network_api::noop::NoopNetwork; - use reth_primitives_traits::{RecoveredBlock, SignerRecoverable}; - use reth_provider::test_utils::{ExtendedAccount, MockEthProvider}; - use reth_rpc_eth_types::cache::cache_new_blocks_task; - use reth_transaction_pool::test_utils::testing_pool; - let provider = MockEthProvider::default(); let tx = TransactionSigned::new_unhashed( TxLegacy { @@ -1311,19 +1286,6 @@ mod tests { #[tokio::test] async fn replay_block_vmtrace_includes_root_and_callcode_bytecode() { - use crate::EthApiBuilder; - use alloy_consensus::{Header, TxLegacy}; - use alloy_primitives::{Signature, TxKind}; - use reth_chain_state::CanonStateNotification; - use reth_ethereum_primitives::{Block, BlockBody, TransactionSigned}; - use reth_evm_ethereum::EthEvmConfig; - use reth_execution_types::{Chain, ExecutionOutcome}; - use reth_network_api::noop::NoopNetwork; - use reth_primitives_traits::{RecoveredBlock, SignerRecoverable}; - use reth_provider::test_utils::{ExtendedAccount, MockEthProvider}; - use reth_rpc_eth_types::cache::cache_new_blocks_task; - use reth_transaction_pool::test_utils::testing_pool; - let provider = MockEthProvider::default(); let target = Address::with_last_byte(0x42); let child = Address::with_last_byte(0x43); diff --git a/crates/stages/types/src/checkpoints.rs b/crates/stages/types/src/checkpoints.rs index 60090cd9715..904477007b9 100644 --- a/crates/stages/types/src/checkpoints.rs +++ b/crates/stages/types/src/checkpoints.rs @@ -652,6 +652,7 @@ stage_unit_checkpoints!( mod tests { use super::*; use alloy_primitives::b256; + use bytes::Buf; use proptest::{collection::vec, option, prelude::*}; use rand::Rng; use reth_codecs::Compact; @@ -809,8 +810,6 @@ mod tests { } fn from_compact(mut buf: &[u8], _len: usize) -> (Self, &[u8]) { - use bytes::Buf; - let last_storage_key = B256::from_slice(&buf[..32]); buf.advance(32); @@ -906,7 +905,6 @@ mod tests { } fn from_compact(mut buf: &[u8], _len: usize) -> (Self, &[u8]) { - use bytes::Buf; let target_block = buf.get_u64(); let last_account_key = B256::from_slice(&buf[..32]); diff --git a/crates/storage/db-api/src/models/integer_list.rs b/crates/storage/db-api/src/models/integer_list.rs index cc5fb8ead17..3da2b7bc9c5 100644 --- a/crates/storage/db-api/src/models/integer_list.rs +++ b/crates/storage/db-api/src/models/integer_list.rs @@ -6,6 +6,7 @@ use core::{fmt, ops::RangeBounds}; use derive_more::Deref; use reth_codecs::DecompressError; use roaring::RoaringTreemap; +use serde::ser::SerializeSeq; /// A data structure that uses Roaring Bitmaps to efficiently store a list of integers. /// @@ -98,8 +99,6 @@ impl serde::Serialize for IntegerList { where S: serde::Serializer, { - use serde::ser::SerializeSeq; - let mut seq = serializer.serialize_seq(Some(self.len() as usize))?; for e in &self.0 { seq.serialize_element(&e)?; diff --git a/crates/storage/db-api/src/models/mod.rs b/crates/storage/db-api/src/models/mod.rs index 677ad4000c6..2677da6865f 100644 --- a/crates/storage/db-api/src/models/mod.rs +++ b/crates/storage/db-api/src/models/mod.rs @@ -254,6 +254,15 @@ add_wrapper_struct!((ClientVersion, CompactClientVersion)); #[cfg(test)] mod tests { + use reth_codecs::{test_utils::UnusedBits, validate_bitflag_backwards_compat}; + use reth_primitives_traits::Account; + use reth_prune_types::{PruneCheckpoint, PruneMode}; + use reth_stages_types::{ + AccountHashingCheckpoint, CheckpointBlockRange, EntitiesCheckpoint, ExecutionCheckpoint, + HeadersCheckpoint, IndexHistoryCheckpoint, StageCheckpoint, StageUnitCheckpoint, + StorageHashingCheckpoint, + }; + // each value in the database has an extra field named flags that encodes metadata about other // fields in the value, e.g. offset and length. // @@ -262,14 +271,6 @@ mod tests { #[test] fn test_ensure_backwards_compatibility() { use super::*; - use reth_codecs::{test_utils::UnusedBits, validate_bitflag_backwards_compat}; - use reth_primitives_traits::Account; - use reth_prune_types::{PruneCheckpoint, PruneMode, PruneSegment}; - use reth_stages_types::{ - AccountHashingCheckpoint, CheckpointBlockRange, EntitiesCheckpoint, - ExecutionCheckpoint, HeadersCheckpoint, IndexHistoryCheckpoint, StageCheckpoint, - StageUnitCheckpoint, StorageHashingCheckpoint, - }; assert_eq!(Account::bitflag_encoded_bytes(), 2); assert_eq!(AccountHashingCheckpoint::bitflag_encoded_bytes(), 1); assert_eq!(CheckpointBlockRange::bitflag_encoded_bytes(), 1); diff --git a/crates/storage/db/Cargo.toml b/crates/storage/db/Cargo.toml index 7a68a7ec2d7..ab18440338e 100644 --- a/crates/storage/db/Cargo.toml +++ b/crates/storage/db/Cargo.toml @@ -67,7 +67,7 @@ proptest.workspace = true assert_matches.workspace = true [features] -dst = ["reth-libmdbx?/dst"] +fuzz = ["reth-libmdbx?/fuzz"] default = ["mdbx"] mdbx = [ "dep:reth-libmdbx", diff --git a/crates/storage/db/src/implementation/mdbx/mod.rs b/crates/storage/db/src/implementation/mdbx/mod.rs index 5c6e7dd4a8e..f879a6de1e1 100644 --- a/crates/storage/db/src/implementation/mdbx/mod.rs +++ b/crates/storage/db/src/implementation/mdbx/mod.rs @@ -267,6 +267,8 @@ pub struct DatabaseEnv { dbis: Arc, /// Cache for metric handles. If `None`, metrics are not recorded. metrics: Option>, + /// Disables long-lived read safety on transactions opened through this handle. + disable_long_read_transaction_safety: bool, /// Write lock for when dealing with a read-write environment. _lock_file: Option, } @@ -314,21 +316,29 @@ impl Database for DatabaseEnv { type TXMut = tx::Tx; fn tx(&self) -> Result { - Tx::new( + let mut tx = Tx::new( self.inner.begin_ro_txn().map_err(|e| DatabaseError::InitTx(e.into()))?, self.dbis.clone(), self.metrics.clone(), ) - .map_err(|e| DatabaseError::InitTx(e.into())) + .map_err(|e| DatabaseError::InitTx(e.into()))?; + if self.disable_long_read_transaction_safety { + tx.disable_long_read_transaction_safety(); + } + Ok(tx) } fn tx_mut(&self) -> Result { - Tx::new( + let mut tx = Tx::new( self.inner.begin_rw_txn().map_err(|e| DatabaseError::InitTx(e.into()))?, self.dbis.clone(), self.metrics.clone(), ) - .map_err(|e| DatabaseError::InitTx(e.into())) + .map_err(|e| DatabaseError::InitTx(e.into()))?; + if self.disable_long_read_transaction_safety { + tx.disable_long_read_transaction_safety(); + } + Ok(tx) } fn path(&self) -> PathBuf { @@ -591,6 +601,7 @@ impl DatabaseEnv { path: path.to_path_buf(), dbis: Arc::default(), metrics: None, + disable_long_read_transaction_safety: false, _lock_file, }; @@ -722,6 +733,15 @@ impl DatabaseEnv { Ok(()) } + + /// Disables timeouts and long-read backtrace reporting for transactions opened through this + /// handle. Existing transactions and other previously cloned handles are unaffected. + /// + /// Use for offline work that intentionally holds transactions open. + pub const fn with_disable_long_read_transaction_safety(mut self) -> Self { + self.disable_long_read_transaction_safety = true; + self + } } impl Deref for DatabaseEnv { diff --git a/crates/storage/db/src/implementation/mdbx/tx.rs b/crates/storage/db/src/implementation/mdbx/tx.rs index 84997783cc7..fd7dc99582c 100644 --- a/crates/storage/db/src/implementation/mdbx/tx.rs +++ b/crates/storage/db/src/implementation/mdbx/tx.rs @@ -221,7 +221,7 @@ struct MetricsHandler { env_metrics: Arc, /// Backtrace of the location where the transaction has been opened. Reported only with debug /// assertions outside DST, because capturing every transaction opening is expensive. - #[cfg(all(debug_assertions, not(feature = "dst")))] + #[cfg(all(debug_assertions, not(feature = "fuzz")))] open_backtrace: Backtrace, _marker: PhantomData, } @@ -236,7 +236,7 @@ impl MetricsHandler { close_recorded: false, record_backtrace: true, backtrace_recorded: AtomicBool::new(false), - #[cfg(all(debug_assertions, not(feature = "dst")))] + #[cfg(all(debug_assertions, not(feature = "fuzz")))] open_backtrace: Backtrace::force_capture(), env_metrics, _marker: PhantomData, @@ -278,9 +278,9 @@ impl MetricsHandler { if now >= self.long_transaction_deadline { let open_duration = now - self.start; self.backtrace_recorded.store(true, Ordering::Relaxed); - #[cfg(all(debug_assertions, not(feature = "dst")))] + #[cfg(all(debug_assertions, not(feature = "fuzz")))] let open_backtrace = format_args!("{}", self.open_backtrace); - #[cfg(not(all(debug_assertions, not(feature = "dst"))))] + #[cfg(not(all(debug_assertions, not(feature = "fuzz"))))] let open_backtrace = tracing::field::Empty; warn!( target: "storage::db::mdbx", @@ -502,12 +502,14 @@ mod tests { .with_max_read_transaction_duration(Some(MaxReadTransactionDuration::Set( MAX_DURATION, ))); - let db = DatabaseEnv::open(dir.path(), DatabaseEnvKind::RW, args).unwrap().with_metrics(); + let db = DatabaseEnv::open(dir.path(), DatabaseEnvKind::RW, args) + .unwrap() + .with_metrics() + .with_disable_long_read_transaction_safety(); let mut tx = db.tx().unwrap(); let metrics_handler = tx.metrics_handler.as_mut().unwrap(); metrics_handler.long_transaction_deadline = metrics_handler.start + MAX_DURATION; - tx.disable_long_read_transaction_safety(); // Give the `TxnManager` some time to time out the transaction. sleep(MAX_DURATION + Duration::from_millis(100)); diff --git a/crates/storage/db/src/mdbx.rs b/crates/storage/db/src/mdbx.rs index 06860f884b0..3a6f7a1dc3e 100644 --- a/crates/storage/db/src/mdbx.rs +++ b/crates/storage/db/src/mdbx.rs @@ -6,6 +6,7 @@ use std::path::Path; use tracing::{info, warn}; pub use crate::implementation::mdbx::*; +use crate::version::{check_db_version_file, create_db_version_file, DatabaseVersionError}; pub use reth_libmdbx::*; /// Tables that have been removed from the schema but may still exist on disk from previous @@ -76,8 +77,6 @@ fn is_zfs(_path: &Path) -> std::io::Result { /// Creates a new database at the specified path if it doesn't exist. Does NOT create tables. Check /// [`init_db`]. pub fn create_db>(path: P, args: DatabaseArguments) -> eyre::Result { - use crate::version::{check_db_version_file, create_db_version_file, DatabaseVersionError}; - let rpath = path.as_ref(); warn_if_zfs(rpath); diff --git a/crates/storage/libmdbx-rs/Cargo.toml b/crates/storage/libmdbx-rs/Cargo.toml index 9bdf3901ddd..b4debbdade2 100644 --- a/crates/storage/libmdbx-rs/Cargo.toml +++ b/crates/storage/libmdbx-rs/Cargo.toml @@ -26,7 +26,7 @@ tracing.workspace = true dashmap = { workspace = true, features = ["inline"], optional = true } [features] -dst = [] +fuzz = [] default = [] return-borrowed = [] read-tx-timeouts = ["dep:dashmap"] diff --git a/crates/storage/libmdbx-rs/src/transaction.rs b/crates/storage/libmdbx-rs/src/transaction.rs index 496f76c2c50..f1326ac6c76 100644 --- a/crates/storage/libmdbx-rs/src/transaction.rs +++ b/crates/storage/libmdbx-rs/src/transaction.rs @@ -577,9 +577,7 @@ impl TransactionPtr { /// Acquires the inner transaction lock to guarantee exclusive access to the transaction /// pointer. fn lock(&self) -> MutexGuard<'_, ()> { - if let Some(lock) = self.lock.try_lock() { - lock - } else { + self.lock.try_lock().unwrap_or_else(|| { tracing::trace!( target: "libmdbx", txn = %self.txn as usize, @@ -588,7 +586,7 @@ impl TransactionPtr { To display the full backtrace, run with `RUST_BACKTRACE=full` env variable." ); self.lock.lock() - } + }) } /// Executes the given closure once the lock on the transaction is acquired. diff --git a/crates/storage/libmdbx-rs/src/txn_manager.rs b/crates/storage/libmdbx-rs/src/txn_manager.rs index 6251ccb23a4..ece338b2698 100644 --- a/crates/storage/libmdbx-rs/src/txn_manager.rs +++ b/crates/storage/libmdbx-rs/src/txn_manager.rs @@ -210,7 +210,7 @@ mod read_transactions { ( tx, Instant::now(), - cfg!(all(debug_assertions, not(feature = "dst"))) + cfg!(all(debug_assertions, not(feature = "fuzz"))) .then(|| Arc::new(Backtrace::force_capture())), ), ); @@ -367,7 +367,7 @@ mod read_transactions { assert!(read_transactions.active.contains_key(&tx_ptr)); assert_eq!( read_transactions.active.get(&tx_ptr).unwrap().2.is_some(), - cfg!(all(debug_assertions, not(feature = "dst"))), + cfg!(all(debug_assertions, not(feature = "fuzz"))), ); tx.open_db(None).unwrap(); diff --git a/crates/storage/provider/src/changesets_utils/state_reverts.rs b/crates/storage/provider/src/changesets_utils/state_reverts.rs index 5145fd1e3c1..2240c7403fb 100644 --- a/crates/storage/provider/src/changesets_utils/state_reverts.rs +++ b/crates/storage/provider/src/changesets_utils/state_reverts.rs @@ -1,6 +1,6 @@ use alloy_primitives::{B256, U256}; use reth_trie::bundle_state::RevertToSlot; -use std::iter::Peekable; +use std::{cmp::Ordering, iter::Peekable}; /// Iterator over storage reverts. /// See [`StorageRevertsIter::next`] for more details. @@ -47,7 +47,6 @@ where match (self.reverts.peek(), self.wiped.peek()) { (Some(revert), Some(wiped)) => { // Compare the keys and return the lesser. - use std::cmp::Ordering; match revert.0.cmp(&wiped.0) { Ordering::Less => self.next_revert(), Ordering::Greater => self.next_wiped(), diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index babacb6e38c..f8acc40a164 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -343,8 +343,7 @@ impl StateRangeProvider for HistoricalStateRangeView { fn account_range_proof(&self, keys: &[B256]) -> ProviderResult> { let multiproof = Proof::new(&self.provider, &self.provider) - .multiproof(MultiProofTargets::accounts(keys.iter().copied())) - .map_err(ProviderError::from)?; + .multiproof(MultiProofTargets::accounts(keys.iter().copied()))?; Ok(multiproof .account_subtree .into_nodes_sorted() @@ -359,8 +358,7 @@ impl StateRangeProvider for HistoricalStateRangeView { keys: &[B256], ) -> ProviderResult> { let multiproof = StorageProof::new_hashed(&self.provider, &self.provider, hashed_address) - .storage_multiproof(keys.iter().copied().collect()) - .map_err(ProviderError::from)?; + .storage_multiproof(keys.iter().copied().collect())?; Ok(multiproof.subtree.into_nodes_sorted().into_iter().map(|(_, bytes)| bytes).collect()) } } @@ -379,6 +377,12 @@ impl DatabaseProviderFactory for BlockchainProvider { } } +impl AsRef for BlockchainProvider { + fn as_ref(&self) -> &N::DB { + self.database.db_ref() + } +} + impl StaticFileProviderFactory for BlockchainProvider { fn static_file_provider(&self) -> StaticFileProvider { self.database.static_file_provider() @@ -1087,8 +1091,8 @@ mod tests { BlockReaderIdExt, BlockSource, ChangeSetReader, DBProvider, DatabaseProviderFactory, HashingWriter, HeaderProvider, RangeEnd, ReceiptProvider, ReceiptProviderIdExt, StageCheckpointWriter, StateProviderFactory, StateRangeProvider, StateRangeProviderFactory, - StateRootProvider, StateWriteConfig, StateWriter, StorageRootProvider, TransactionVariant, - TransactionsProvider, + StateRootProvider, StateWriteConfig, StateWriter, StorageRootProvider, StorageSettings, + StorageSettingsCache, TransactionVariant, TransactionsProvider, TrieWriter, }; use reth_testing_utils::generators::{ self, random_block, random_block_range, random_changeset_range, random_eoa_accounts, @@ -1096,8 +1100,9 @@ mod tests { }; use reth_trie::{ bundle_state::{BundleState, OriginalValuesKnown}, + test_utils::TrieTestHarness, updates::TrieUpdates, - HashedPostState, HashedStorage, SortedTrieData, + HashedPostState, HashedStorage, MultiProofTargetsV2, SortedTrieData, }; use std::{ collections::{BTreeMap, HashMap}, @@ -1108,8 +1113,6 @@ mod tests { #[test] fn historical_proofs_complete_masked_trie_rows() -> eyre::Result<()> { use crate::{StaticFileProviderFactory, StaticFileSegment, StaticFileWriter}; - use reth_storage_api::{StorageSettings, StorageSettingsCache, TrieWriter}; - use reth_trie::{test_utils::TrieTestHarness, MultiProofTargetsV2}; let address = Address::with_last_byte(1); let other_address = Address::with_last_byte(2); diff --git a/crates/storage/provider/src/providers/consistent.rs b/crates/storage/provider/src/providers/consistent.rs index b4f05b0c684..1345260e268 100644 --- a/crates/storage/provider/src/providers/consistent.rs +++ b/crates/storage/provider/src/providers/consistent.rs @@ -1417,20 +1417,28 @@ mod tests { test_utils::create_test_provider_factory, BlockWriter, }; use alloy_eips::BlockHashOrNumber; - use alloy_primitives::B256; + use alloy_primitives::{B256, U256}; use itertools::Itertools; use rand::Rng; use reth_chain_state::{ExecutedBlock, NewCanonicalChain}; - use reth_db_api::models::AccountBeforeTx; + use reth_db_api::{ + models::{AccountBeforeTx, StorageSettings}, + tables, + transaction::DbTxMut, + }; use reth_ethereum_primitives::Block; use reth_execution_types::{BlockExecutionOutput, BlockExecutionResult, ExecutionOutcome}; - use reth_primitives_traits::{RecoveredBlock, SealedBlock}; - use reth_storage_api::{BlockReader, BlockSource, ChangeSetReader, StateReader}; + use reth_primitives_traits::{RecoveredBlock, SealedBlock, StorageEntry}; + use reth_storage_api::{ + BlockReader, BlockSource, ChangeSetReader, StateReader, StorageChangeSetReader, + StorageSettingsCache, + }; use reth_testing_utils::generators::{ self, random_block_range, random_changeset_range, random_eoa_accounts, BlockRangeParams, }; use reth_trie::bundle_state::BundleState; use std::{ + collections::HashMap, ops::{Bound, Range, RangeBounds}, sync::Arc, }; @@ -1815,12 +1823,6 @@ mod tests { } #[test] fn test_get_state_storage_value_plain_state() -> eyre::Result<()> { - use alloy_primitives::U256; - use reth_db_api::{models::StorageSettings, tables, transaction::DbTxMut}; - use reth_primitives_traits::StorageEntry; - use reth_storage_api::StorageSettingsCache; - use std::collections::HashMap; - let address = alloy_primitives::Address::with_last_byte(1); let account = reth_primitives_traits::Account { nonce: 1, @@ -1897,11 +1899,6 @@ mod tests { #[test] fn test_storage_changeset_consistent_keys_plain_state() -> eyre::Result<()> { - use alloy_primitives::U256; - use reth_db_api::models::StorageSettings; - use reth_storage_api::{StorageChangeSetReader, StorageSettingsCache}; - use std::collections::HashMap; - let mut rng = generators::rng(); let factory = create_test_provider_factory(); factory.set_storage_settings_cache(StorageSettings::v1()); @@ -1996,11 +1993,6 @@ mod tests { #[test] fn test_storage_changesets_range_consistent_keys_plain_state() -> eyre::Result<()> { - use alloy_primitives::U256; - use reth_db_api::models::StorageSettings; - use reth_storage_api::{StorageChangeSetReader, StorageSettingsCache}; - use std::collections::HashMap; - let mut rng = generators::rng(); let factory = create_test_provider_factory(); factory.set_storage_settings_cache(StorageSettings::v1()); diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 540ec4acb48..3d172c86d8f 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -4093,13 +4093,14 @@ mod tests { BlockWriter, }; use alloy_consensus::Header; + use alloy_eips::BlockNumHash; use alloy_primitives::{ map::{AddressMap, B256Map}, U256, }; use evm2::evm::AccountInfo; use reth_chain_state::{test_utils::TestBlockBuilder, ExecutedBlock}; - use reth_db_api::models::StorageSettings; + use reth_db_api::models::{SnapAttempt, StorageSettings}; use reth_ethereum_primitives::Receipt; use reth_execution_types::{AccountRevertInit, BlockExecutionOutput, BlockExecutionResult}; use reth_primitives_traits::SealedBlock; @@ -4133,9 +4134,6 @@ mod tests { #[test] fn snap_attempt_guards_finish_checkpoint_writers() { - use alloy_eips::BlockNumHash; - use reth_db_api::models::SnapAttempt; - let factory = create_test_provider_factory(); let provider = factory.database_provider_rw().unwrap(); provider.update_pipeline_stages(5, false).unwrap(); @@ -4187,9 +4185,6 @@ mod tests { #[test] fn snap_attempt_guards_partial_state_trie_advances() { - use alloy_eips::BlockNumHash; - use reth_db_api::models::SnapAttempt; - for abandoned in [false, true] { let factory = create_test_provider_factory(); let provider = factory.database_provider_rw().unwrap(); diff --git a/crates/storage/provider/src/providers/rocksdb/invariants.rs b/crates/storage/provider/src/providers/rocksdb/invariants.rs index d069a7889b6..864de083694 100644 --- a/crates/storage/provider/src/providers/rocksdb/invariants.rs +++ b/crates/storage/provider/src/providers/rocksdb/invariants.rs @@ -484,15 +484,21 @@ mod tests { BlockWriter, DatabaseProviderFactory, RocksDBProviderFactory, StageCheckpointWriter, TransactionsProvider, }; - use alloy_primitives::{Address, B256}; + use alloy_primitives::{Address, B256, U256}; use reth_chainspec::MAINNET; - use reth_db::cursor::{DbCursorRO, DbCursorRW}; + use reth_db::{ + cursor::{DbCursorRO, DbCursorRW}, + models::AccountBeforeTx, + }; use reth_db_api::{ - models::{storage_sharded_key::StorageShardedKey, StorageSettings}, + models::{ + storage_sharded_key::StorageShardedKey, ShardedKey, StorageBeforeTx, StorageSettings, + }, tables::{self, BlockNumberList}, transaction::DbTxMut, }; use reth_stages_types::StageCheckpoint; + use reth_static_file_types::StaticFileSegment; use reth_testing_utils::generators::{self, BlockRangeParams}; use tempfile::TempDir; @@ -931,8 +937,6 @@ mod tests { #[test] fn test_check_consistency_accounts_history_sentinel_only_with_checkpoint_is_first_run() { - use reth_db_api::models::ShardedKey; - let temp_dir = TempDir::new().unwrap(); let rocksdb = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); @@ -1139,10 +1143,6 @@ mod tests { #[test] fn test_check_consistency_accounts_history_sf_tip_equals_checkpoint_no_action() { - use reth_db::models::AccountBeforeTx; - use reth_db_api::models::ShardedKey; - use reth_static_file_types::StaticFileSegment; - let temp_dir = TempDir::new().unwrap(); let rocksdb = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); @@ -1236,9 +1236,6 @@ mod tests { /// 7. Verify stale entries for blocks > 5000 are pruned and batching worked #[test] fn test_check_consistency_storages_history_heals_via_changesets_large_range() { - use alloy_primitives::U256; - use reth_db_api::models::StorageBeforeTx; - const TOTAL_BLOCKS: u64 = 15_000; const CHECKPOINT_BLOCK: u64 = 5_000; @@ -1353,9 +1350,6 @@ mod tests { /// This catches off-by-one bugs where checkpoint block data is incorrectly deleted. #[test] fn test_check_consistency_storages_history_preserves_checkpoint_block() { - use alloy_primitives::U256; - use reth_db_api::models::StorageBeforeTx; - const CHECKPOINT_BLOCK: u64 = 100; const SF_TIP: u64 = 200; @@ -1453,10 +1447,6 @@ mod tests { /// - The batching worked (no OOM, completed successfully) #[test] fn test_check_consistency_accounts_history_heals_via_changesets_large_range() { - use reth_db::models::AccountBeforeTx; - use reth_db_api::models::ShardedKey; - use reth_static_file_types::StaticFileSegment; - let temp_dir = TempDir::new().unwrap(); let rocksdb = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); @@ -1566,9 +1556,6 @@ mod tests { /// Tests that accounts history healing preserves entries at exactly the checkpoint block. #[test] fn test_check_consistency_accounts_history_preserves_checkpoint_block() { - use reth_db::models::AccountBeforeTx; - use reth_db_api::models::ShardedKey; - const CHECKPOINT_BLOCK: u64 = 100; const SF_TIP: u64 = 200; @@ -1638,10 +1625,6 @@ mod tests { #[test] fn test_check_consistency_storages_history_sf_tip_equals_checkpoint_no_action() { - use alloy_primitives::U256; - use reth_db::models::StorageBeforeTx; - use reth_static_file_types::StaticFileSegment; - let temp_dir = TempDir::new().unwrap(); let rocksdb = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); diff --git a/crates/storage/provider/src/providers/rocksdb/provider.rs b/crates/storage/provider/src/providers/rocksdb/provider.rs index a441425af35..7584f15e03c 100644 --- a/crates/storage/provider/src/providers/rocksdb/provider.rs +++ b/crates/storage/provider/src/providers/rocksdb/provider.rs @@ -2420,14 +2420,12 @@ impl<'a> RocksDBBatch<'a> { // 3. Invalid (no more keys) // If the current key's prefix >= our target prefix, we may be able to skip the seek. let needs_seek = if iter.valid() { - if let Some(current_key) = iter.key() { - // If current key's prefix < target prefix, we need to seek forward - // If current key's prefix > target prefix, this target has no shards (skip) - // If current key's prefix == target prefix, we're already positioned + // If current key's prefix < target prefix, we need to seek forward + // If current key's prefix > target prefix, this target has no shards (skip) + // If current key's prefix == target prefix, we're already positioned + iter.key().is_none_or(|current_key| { current_key.get(..PREFIX_LEN).is_none_or(|p| p < target_prefix) - } else { - true - } + }) } else { true }; @@ -2576,14 +2574,12 @@ impl<'a> RocksDBBatch<'a> { // 3. Invalid (no more keys) // If the current key's prefix >= our target prefix, we may be able to skip the seek. let needs_seek = if iter.valid() { - if let Some(current_key) = iter.key() { - // If current key's prefix < target prefix, we need to seek forward - // If current key's prefix > target prefix, this target has no shards (skip) - // If current key's prefix == target prefix, we're already positioned + // If current key's prefix < target prefix, we need to seek forward + // If current key's prefix > target prefix, this target has no shards (skip) + // If current key's prefix == target prefix, we're already positioned + iter.key().is_none_or(|current_key| { current_key.get(..PREFIX_LEN).is_none_or(|p| p < target_prefix) - } else { - true - } + }) } else { true }; @@ -3185,7 +3181,10 @@ mod tests { table::Table, tables, }; - use rocksdb::perf::{set_perf_stats, PerfContext, PerfMetric, PerfStatsLevel}; + use rocksdb::{ + perf::{set_perf_stats, PerfContext, PerfMetric, PerfStatsLevel}, + statistics::{Histogram, Ticker}, + }; use tempfile::TempDir; /// Number of deleted keys `RocksDB` skipped on this thread while running `f`. @@ -3518,8 +3517,6 @@ mod tests { /// levels the C API defines, so the variant name alone does not tell us what was selected. #[test] fn test_statistics_level_skips_timers() { - use rocksdb::statistics::{Histogram, Ticker}; - let temp_dir = TempDir::new().unwrap(); let cache = Cache::new_lru_cache(1 << 20); let options = RocksDBBuilder::default_options(rocksdb::LogLevel::Info, &cache, true); diff --git a/crates/storage/provider/src/providers/static_file/manager.rs b/crates/storage/provider/src/providers/static_file/manager.rs index 296f9c53450..d5bb747f294 100644 --- a/crates/storage/provider/src/providers/static_file/manager.rs +++ b/crates/storage/provider/src/providers/static_file/manager.rs @@ -3181,6 +3181,9 @@ mod tests { use super::StaticFileWriter; use crate::{providers::StaticFileProvider, BlockHashReader, StaticFileProviderBuilder}; + use alloy_consensus::{SignableTransaction, TxLegacy}; + use alloy_primitives::Signature; + use reth_db::{tables, transaction::DbTxMut}; #[test] fn recovery_prunes_transaction_zero_at_empty_genesis() -> eyre::Result<()> { @@ -3188,9 +3191,6 @@ mod tests { test_utils::create_test_provider_factory, StaticFileProviderFactory, TransactionsProvider, }; - use alloy_consensus::{SignableTransaction, TxLegacy}; - use alloy_primitives::Signature; - use reth_db::{tables, transaction::DbTxMut}; let factory = create_test_provider_factory(); let provider = factory.provider_rw()?; diff --git a/crates/storage/provider/src/providers/static_file/writer.rs b/crates/storage/provider/src/providers/static_file/writer.rs index eb86e2823fd..36398391d43 100644 --- a/crates/storage/provider/src/providers/static_file/writer.rs +++ b/crates/storage/provider/src/providers/static_file/writer.rs @@ -20,6 +20,7 @@ use std::{ borrow::Borrow, cmp::Ordering, fmt::Debug, + fs::OpenOptions, path::{Path, PathBuf}, sync::{Arc, Weak}, }; @@ -455,7 +456,6 @@ impl StaticFileProviderRW { // Truncate sidecar file if it has invalid blocks if actual_sidecar_blocks > correct_blocks { - use std::fs::OpenOptions; let file = OpenOptions::new() .write(true) .open(&csoff_path) diff --git a/crates/storage/storage-overlay/src/builder.rs b/crates/storage/storage-overlay/src/builder.rs index 3fdce81c17e..9e6f7f5d3c4 100644 --- a/crates/storage/storage-overlay/src/builder.rs +++ b/crates/storage/storage-overlay/src/builder.rs @@ -24,6 +24,7 @@ use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted, TrieInputSort use reth_trie_common::bundle_state::BundleState; use reth_trie_db::DatabaseHashedPostState; use std::{ + io::Error, ops::RangeInclusive, sync::Arc, time::{Duration, Instant}, @@ -265,8 +266,6 @@ impl OverlayBuilder { where Provider: BlockNumReader + PruneCheckpointReader, { - use std::io::Error; - let mut in_mem_chain = self .parent_state .iter() @@ -441,7 +440,11 @@ impl OverlayBuilder { "Collecting trie reverts for overlay state provider" ); - let trie_reverts = if trie_changesets { + // Trie updates applied directly over the on-disk trie. With trie changesets these + // are exact reverts down to the anchor's trie; without them they are the in-memory + // nodes up to Finish (or nothing when the trie is already at Finish), and the + // prefix sets cover every key that differs from the anchor. + let base_trie_updates = if trie_changesets { let _guard = debug_span!(target: "storage::overlay", "retrieving_trie_reverts") .entered(); let start = Instant::now(); @@ -497,14 +500,20 @@ impl OverlayBuilder { let (overlay_trie, overlay_state) = self.resolve_state_trie_overlays(anchor.hash)?; - let trie_updates = if trie_reverts.is_empty() { + // Without trie changesets the base trie updates describe Finish, while the overlay + // trie nodes are relative to the anchor's trie. Mixing them can leave Finish-only + // nodes beneath anchor-relative branches, so only the prefix sets may describe the + // in-memory blocks. + let trie_updates = if !trie_changesets { + base_trie_updates + } else if base_trie_updates.is_empty() { overlay_trie } else if !overlay_trie.is_empty() { - let mut trie_reverts = (*trie_reverts).clone(); - trie_reverts.extend_ref_and_sort(&overlay_trie); - Arc::new(trie_reverts) + let mut base_trie_updates = (*base_trie_updates).clone(); + base_trie_updates.extend_ref_and_sort(&overlay_trie); + Arc::new(base_trie_updates) } else { - trie_reverts + base_trie_updates }; let hashed_state_updates = if hashed_state_reverts.is_empty() { @@ -518,8 +527,7 @@ impl OverlayBuilder { // Without trie changesets the trie nodes still describe Finish, so every key whose // value differs from Finish must be invalidated: both the reverted keys and the - // keys changed by the in-memory blocks replayed on top of the anchor. The overlay - // trie nodes are only valid relative to the anchor's trie, not Finish's. + // keys changed by the in-memory blocks replayed on top of the anchor. let prefix_sets = if trie_changesets { Default::default() } else { @@ -1173,6 +1181,9 @@ mod tests { .build_state_trie_overlay(&provider, false) .unwrap(); + // The fork's trie nodes are relative to the anchor, not to the Finish trie on disk. + assert!(account_node_paths(&overlay).is_empty()); + let mut prefix_sets = overlay.input().prefix_sets.clone().freeze(); assert!(prefix_sets .account_prefix_set diff --git a/crates/storage/storage-overlay/src/manager.rs b/crates/storage/storage-overlay/src/manager.rs index dd955556c87..040f126b4de 100644 --- a/crates/storage/storage-overlay/src/manager.rs +++ b/crates/storage/storage-overlay/src/manager.rs @@ -308,7 +308,7 @@ impl OverlayManager { worker_pool.spawn(move || { let _span = tracing::trace_span!( target: "storage::overlay::manager", - parent: parent_span, + parent: &parent_span, "precompute_execution_overlay", %tip_hash, %anchor_hash, diff --git a/crates/tasks/src/for_each_ordered.rs b/crates/tasks/src/for_each_ordered.rs index 4f5eca471e5..b44be4385ea 100644 --- a/crates/tasks/src/for_each_ordered.rs +++ b/crates/tasks/src/for_each_ordered.rs @@ -1,7 +1,10 @@ use crossbeam_utils::CachePadded; use parking_lot::{Condvar, Mutex}; use rayon::iter::{IndexedParallelIterator, ParallelIterator}; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::{ + panic::{catch_unwind, AssertUnwindSafe}, + sync::atomic::{AtomicBool, Ordering}, +}; /// Extension trait for [`IndexedParallelIterator`] /// that streams results to a sequential consumer in index order. @@ -111,8 +114,6 @@ where I::Item: Send, F: FnMut(I::Item), { - use std::panic::{catch_unwind, AssertUnwindSafe}; - let n = iter.len(); if n == 0 { return; @@ -165,9 +166,12 @@ where mod tests { use super::*; use rayon::prelude::*; - use std::sync::{ - atomic::{AtomicUsize, Ordering}, - Barrier, + use std::{ + rc::Rc, + sync::{ + atomic::{AtomicUsize, Ordering}, + Barrier, + }, }; #[test] @@ -265,7 +269,6 @@ mod tests { #[test] fn callback_is_not_send() { - use std::rc::Rc; let counter = Rc::new(std::cell::Cell::new(0u64)); let input: Vec = (0..100).collect(); input.par_iter().map(|&x| x).for_each_ordered(|x| { diff --git a/crates/tracing-otlp/Cargo.toml b/crates/tracing-otlp/Cargo.toml index 4274df1e26f..318cb973b6f 100644 --- a/crates/tracing-otlp/Cargo.toml +++ b/crates/tracing-otlp/Cargo.toml @@ -9,10 +9,15 @@ repository.workspace = true exclude.workspace = true [dependencies] +# reth +reth-fs-util = { workspace = true, optional = true } + # obs opentelemetry_sdk = { workspace = true, optional = true } opentelemetry = { workspace = true, optional = true } opentelemetry-otlp = { workspace = true, optional = true, features = ["grpc-tonic", "reqwest-rustls"] } +opentelemetry-proto = { workspace = true, optional = true, features = ["gen-tonic-messages", "trace"] } +prost = { workspace = true, optional = true } opentelemetry-semantic-conventions = { workspace = true, optional = true } opentelemetry-appender-tracing = { workspace = true, optional = true } tracing-opentelemetry = { workspace = true, optional = true } @@ -23,7 +28,7 @@ tracing = { workspace = true, optional = true } base64 = { version = "0.23", optional = true } clap = { workspace = true, features = ["derive"] } eyre.workspace = true -url.workspace = true +url = { workspace = true, features = ["std"] } [lints] workspace = true @@ -38,6 +43,9 @@ otlp = [ "tracing", "tracing-subscriber", "base64", + "reth-fs-util", + "opentelemetry-proto", + "prost", ] otlp-logs = [ diff --git a/crates/tracing-otlp/src/file.rs b/crates/tracing-otlp/src/file.rs new file mode 100644 index 00000000000..39085118343 --- /dev/null +++ b/crates/tracing-otlp/src/file.rs @@ -0,0 +1,106 @@ +//! Local length-delimited OTLP trace batch exporter. + +use crate::trace_file_path; +use eyre::WrapErr; +use opentelemetry_proto::{ + tonic::collector::trace::v1::ExportTraceServiceRequest, + transform::{ + common::tonic::ResourceAttributesWithSchema, + trace::tonic::group_spans_by_resource_and_scope, + }, +}; +use opentelemetry_sdk::{ + error::{OTelSdkError, OTelSdkResult}, + trace::{SpanData, SpanExporter}, + Resource, +}; +use prost::Message; +use reth_fs_util::FsPathError; +use std::{ + fs::{File, OpenOptions}, + io::{BufReader, Read, Seek, SeekFrom, Write}, + sync::Mutex, +}; +use url::Url; + +#[cfg(unix)] +use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; + +#[derive(Debug)] +pub(crate) struct FileSpanExporter { + file: Mutex>, + resource: ResourceAttributesWithSchema, +} + +impl FileSpanExporter { + pub(crate) fn new(endpoint: &Url) -> eyre::Result { + let path = trace_file_path(endpoint)?; + let mut options = OpenOptions::new(); + options.create(true).read(true).append(true); + #[cfg(unix)] + options.mode(0o600); + let file = options.open(&path).map_err(|err| FsPathError::open(err, &path))?; + // Readers can acquire advisory locks too, so reject shared existing files. + #[cfg(unix)] + eyre::ensure!( + file.metadata()?.permissions().mode() & 0o077 == 0, + "Trace file {path:?} must be owner-only; restrict its permissions to 0600" + ); + // Keep the lock for the exporter's lifetime so partial writes cannot interleave. + file.try_lock() + .wrap_err_with(|| format!("Could not exclusively lock trace file {path:?}"))?; + // Never append after an incomplete record left by an interrupted writer. + // Inspect framing without allocating or decoding entire trace batches. + let length = file.metadata()?.len(); + let mut reader = BufReader::new(&file); + while reader.stream_position()? < length { + let mut delimiter = [0u8; 10]; + let mut used = 0; + for byte in &mut delimiter { + reader + .read_exact(std::slice::from_mut(byte)) + .wrap_err("Incomplete OTLP file length delimiter")?; + used += 1; + if *byte & 0x80 == 0 { + break; + } + } + let payload = prost::decode_length_delimiter(&delimiter[..used])?; + let end = reader + .stream_position()? + .checked_add(u64::try_from(payload)?) + .filter(|end| *end <= length) + .ok_or_else(|| eyre::eyre!("Incomplete OTLP file batch"))?; + reader.seek(SeekFrom::Start(end))?; + } + Ok(Self { + file: Mutex::new(Some(file)), + resource: (&Resource::builder_empty().build()).into(), + }) + } +} + +impl SpanExporter for FileSpanExporter { + async fn export(&self, batch: Vec) -> OTelSdkResult { + let request = ExportTraceServiceRequest { + resource_spans: group_spans_by_resource_and_scope(batch, &self.resource), + }; + let bytes = request.encode_length_delimited_to_vec(); + let mut file = + self.file.lock().map_err(|err| OTelSdkError::InternalFailure(err.to_string()))?; + let writer = file.as_mut().ok_or_else(|| { + OTelSdkError::InternalFailure("Trace file disabled after a write failure".into()) + })?; + let result = writer.write_all(&bytes).and_then(|()| writer.flush()); + if result.is_err() { + // A failed write may have committed a partial record. Do not append + // another batch after it, including on a later SDK retry. + file.take(); + } + result.map_err(|err| OTelSdkError::InternalFailure(err.to_string())) + } + + fn set_resource(&mut self, resource: &Resource) { + self.resource = resource.into(); + } +} diff --git a/crates/tracing-otlp/src/lib.rs b/crates/tracing-otlp/src/lib.rs index 69c17145663..7a91382c729 100644 --- a/crates/tracing-otlp/src/lib.rs +++ b/crates/tracing-otlp/src/lib.rs @@ -8,8 +8,11 @@ use clap::ValueEnum; use eyre::ensure; +use std::path::PathBuf; use url::Url; +#[cfg(feature = "otlp")] +mod file; #[cfg(feature = "otlp")] mod otlp; #[cfg(feature = "otlp")] @@ -34,7 +37,13 @@ impl OtlpProtocol { /// /// For HTTP: Ensures the path ends with `/v1/traces`, appending it if necessary. /// For gRPC: Ensures the path does NOT include `/v1/traces`. + /// For file URLs: Requires a local absolute path without a query or fragment. + /// The protocol is ignored for file export. pub fn validate_endpoint(&self, url: &mut Url) -> eyre::Result<()> { + if url.scheme() == "file" { + trace_file_path(url)?; + return Ok(()); + } self.validate_endpoint_with_path(url, HTTP_TRACE_ENDPOINT) } @@ -43,6 +52,7 @@ impl OtlpProtocol { /// For HTTP: Ensures the path ends with `/v1/logs`, appending it if necessary. /// For gRPC: Ensures the path does NOT include `/v1/logs`. pub fn validate_logs_endpoint(&self, url: &mut Url) -> eyre::Result<()> { + ensure!(url.scheme() != "file", "OTLP logs do not support file endpoints"); self.validate_endpoint_with_path(url, HTTP_LOGS_ENDPOINT) } @@ -66,3 +76,18 @@ impl OtlpProtocol { Ok(()) } } + +/// Validates a local trace file URL without applying network endpoint path rules. +fn trace_file_path(url: &Url) -> eyre::Result { + ensure!(url.scheme() == "file", "OTLP trace file endpoint must use the file scheme"); + ensure!( + url.host_str().is_none() && url.query().is_none() && url.fragment().is_none(), + "OTLP trace file endpoint must be local and have no query or fragment" + ); + let path = url.to_file_path().map_err(|()| eyre::eyre!("Invalid OTLP trace file path"))?; + ensure!( + path.is_absolute() && path.file_name().is_some(), + "OTLP trace file path must be absolute and name a file" + ); + Ok(path) +} diff --git a/crates/tracing-otlp/src/otlp.rs b/crates/tracing-otlp/src/otlp.rs index 91315a68912..996c3056fb5 100644 --- a/crates/tracing-otlp/src/otlp.rs +++ b/crates/tracing-otlp/src/otlp.rs @@ -1,12 +1,12 @@ //! OTLP exporter layers and configuration. -use crate::OtlpProtocol; +use crate::{file::FileSpanExporter, trace_file_path, OtlpProtocol}; use eyre::ensure; use opentelemetry::{global, trace::TracerProvider, KeyValue, Value}; use opentelemetry_otlp::{SpanExporter, WithExportConfig}; use opentelemetry_sdk::{ propagation::TraceContextPropagator, - trace::{Sampler, SdkTracer, SdkTracerProvider}, + trace::{Sampler, SdkTracer}, Resource, }; use opentelemetry_semantic_conventions::{attribute::SERVICE_VERSION, SCHEMA_URL}; @@ -17,6 +17,8 @@ use url::Url; use base64::{prelude::BASE64_STANDARD, Engine}; +pub use opentelemetry_sdk::trace::SdkTracerProvider; + const OTEL_EXPORTER_OTLP_TRACES_HEADERS: &str = "OTEL_EXPORTER_OTLP_TRACES_HEADERS"; const OTEL_EXPORTER_OTLP_LOGS_HEADERS: &str = "OTEL_EXPORTER_OTLP_LOGS_HEADERS"; @@ -25,6 +27,24 @@ const OTEL_EXPORTER_OTLP_LOGS_HEADERS: &str = "OTEL_EXPORTER_OTLP_LOGS_HEADERS"; /// This layer can be added to a [`tracing_subscriber::Registry`] to enable `OpenTelemetry` tracing /// with OTLP export to an url. pub fn span_layer(otlp_config: OtlpConfig) -> eyre::Result> +where + for<'span> S: Subscriber + LookupSpan<'span>, +{ + let (layer, provider) = span_layer_with_provider(otlp_config)?; + global::set_tracer_provider(provider); + Ok(layer) +} + +/// Creates an OTLP span layer and returns its provider for explicit flush and shutdown. +/// +/// Unlike [`span_layer`], this does not install the provider globally. Call +/// [`SdkTracerProvider::force_flush`] to export completed spans and +/// [`SdkTracerProvider::shutdown`] after dropping active spans and the subscriber. +/// File export uses the SDK batch worker and does not require a Tokio runtime. +/// Network transports retain their usual runtime requirements. +pub fn span_layer_with_provider( + otlp_config: OtlpConfig, +) -> eyre::Result<(OpenTelemetryLayer, SdkTracerProvider)> where for<'span> S: Subscriber + LookupSpan<'span>, { @@ -32,35 +52,33 @@ where let resource = build_resource(otlp_config.service_name.clone(), otlp_config.service_version.as_deref()); - - let span_builder = SpanExporter::builder(); - - let span_exporter = match otlp_config.protocol { - OtlpProtocol::Http => { - span_builder.with_http().with_endpoint(otlp_config.endpoint.as_str()).build()? - } - OtlpProtocol::Grpc => { - span_builder.with_tonic().with_endpoint(otlp_config.endpoint.as_str()).build()? - } - }; - - let sampler = build_sampler(otlp_config.sample_ratio)?; - - let tracer_provider = SdkTracerProvider::builder() + let builder = SdkTracerProvider::builder() .with_resource(resource) - .with_sampler(sampler) - .with_batch_exporter(span_exporter) - .build(); - - global::set_tracer_provider(tracer_provider.clone()); + .with_sampler(build_sampler(otlp_config.sample_ratio)?); + + let tracer_provider = if otlp_config.endpoint.scheme() == "file" { + builder.with_batch_exporter(FileSpanExporter::new(&otlp_config.endpoint)?).build() + } else { + let span_builder = SpanExporter::builder(); + let span_exporter = match otlp_config.protocol { + OtlpProtocol::Http => { + span_builder.with_http().with_endpoint(otlp_config.endpoint.as_str()).build()? + } + OtlpProtocol::Grpc => { + span_builder.with_tonic().with_endpoint(otlp_config.endpoint.as_str()).build()? + } + }; + builder.with_batch_exporter(span_exporter).build() + }; let tracer = tracer_provider.tracer(otlp_config.service_name); - Ok(tracing_opentelemetry::layer() + let layer = tracing_opentelemetry::layer() .with_tracer(tracer) .with_location(false) .with_tracked_inactivity(false) .with_target(false) - .with_threads(false)) + .with_threads(false); + Ok((layer, tracer_provider)) } /// Creates a tracing layer that exports logs to an OTLP endpoint. @@ -134,7 +152,11 @@ impl OtlpConfig { ); } - set_otlp_auth_header_from_endpoint(&endpoint, OTEL_EXPORTER_OTLP_TRACES_HEADERS)?; + if endpoint.scheme() == "file" { + trace_file_path(&endpoint)?; + } else { + set_otlp_auth_header_from_endpoint(&endpoint, OTEL_EXPORTER_OTLP_TRACES_HEADERS)?; + } Ok(Self { service_name: service_name.into(), service_version: None, @@ -191,6 +213,7 @@ impl OtlpLogsConfig { endpoint: Url, protocol: OtlpProtocol, ) -> eyre::Result { + ensure!(endpoint.scheme() != "file", "OTLP logs do not support file endpoints"); set_otlp_auth_header_from_endpoint(&endpoint, OTEL_EXPORTER_OTLP_LOGS_HEADERS)?; Ok(Self { service_name: service_name.into(), diff --git a/crates/tracing/src/formatter.rs b/crates/tracing/src/formatter.rs index 1950ccb7795..dd8fb4ec659 100644 --- a/crates/tracing/src/formatter.rs +++ b/crates/tracing/src/formatter.rs @@ -46,11 +46,9 @@ impl LogFormat { where F: Filter + Send + Sync + 'static, { - let ansi = if let Some(color) = color { + let ansi = color.is_some_and(|color| { std::env::var("RUST_LOG_STYLE").map(|val| val != "never").unwrap_or(color != "never") - } else { - false - }; + }); let target = std::env::var("RUST_LOG_TARGET").map(|val| val != "0").unwrap_or(show_target); diff --git a/crates/trie/common/src/key.rs b/crates/trie/common/src/key.rs index 9082fa93787..b2b8a145893 100644 --- a/crates/trie/common/src/key.rs +++ b/crates/trie/common/src/key.rs @@ -20,11 +20,10 @@ impl KeyHasher for KeccakKeyHasher { #[cfg(test)] mod tests { use super::*; + use alloy_primitives::Address; #[test] fn test_keccak_key_hasher_always_hashes_regardless_of_length() { - use alloy_primitives::Address; - let addr = Address::repeat_byte(0x42); assert_eq!(KeccakKeyHasher::hash_key(addr), keccak256(addr)); diff --git a/crates/trie/common/src/proofs.rs b/crates/trie/common/src/proofs.rs index 1f0489d8cff..c974fb69341 100644 --- a/crates/trie/common/src/proofs.rs +++ b/crates/trie/common/src/proofs.rs @@ -57,12 +57,10 @@ impl MultiProofTargets { /// i.e., the values that are in `self` but not in `other`. pub fn retain_difference(&mut self, other: &Self) { self.0.retain(|hashed_address, hashed_slots| { - if let Some(other_hashed_slots) = other.get(hashed_address) { + other.get(hashed_address).is_none_or(|other_hashed_slots| { hashed_slots.retain(|hashed_slot| !other_hashed_slots.contains(hashed_slot)); !hashed_slots.is_empty() - } else { - true - } + }) }); } diff --git a/crates/trie/db/src/lib.rs b/crates/trie/db/src/lib.rs index f9b44133fe7..ca71d01ed67 100644 --- a/crates/trie/db/src/lib.rs +++ b/crates/trie/db/src/lib.rs @@ -10,6 +10,7 @@ mod proof; mod state; mod storage; mod trie_cursor; +mod verify; pub use hashed_cursor::{ DatabaseHashedAccountCursor, DatabaseHashedCursorFactory, DatabaseHashedStorageCursor, @@ -23,6 +24,7 @@ pub use trie_cursor::{ DatabaseAccountTrieCursor, DatabaseStorageTrieCursor, DatabaseTrieCursorFactory, LegacyKeyAdapter, PackedKeyAdapter, StorageTrieEntryLike, TrieKeyAdapter, TrieTableAdapter, }; +pub use verify::DatabaseVerifier; /// Dispatches a trie operation using the correct [`TrieKeyAdapter`] based on storage settings. /// diff --git a/crates/trie/db/src/state.rs b/crates/trie/db/src/state.rs index 476a85f8b2e..524b4125493 100644 --- a/crates/trie/db/src/state.rs +++ b/crates/trie/db/src/state.rs @@ -349,13 +349,16 @@ mod tests { use alloy_primitives::{hex, keccak256, map::HashMap, Address, B256, U256}; use evm2::evm::AccountInfo; use reth_db_api::{ - models::{AccountBeforeTx, BlockNumberAddress}, + models::{AccountBeforeTx, BlockNumberAddress, StorageBeforeTx, StorageSettings}, tables, transaction::DbTxMut, }; use reth_execution_errors::StateRootError; use reth_primitives_traits::{Account, StorageEntry}; - use reth_provider::test_utils::create_test_provider_factory; + use reth_provider::{ + test_utils::create_test_provider_factory, StaticFileProviderFactory, StaticFileSegment, + StaticFileWriter, + }; use reth_storage_api::StorageSettingsCache; use reth_trie::{ HashedPostState, HashedPostStateSorted, HashedStorage, KeccakKeyHasher, StateRoot, @@ -536,9 +539,6 @@ mod tests { #[test] fn from_reverts_with_hashed_state() { - use reth_db_api::models::{StorageBeforeTx, StorageSettings}; - use reth_provider::{StaticFileProviderFactory, StaticFileSegment, StaticFileWriter}; - let factory = create_test_provider_factory(); factory.set_storage_settings_cache(StorageSettings::v2()); diff --git a/crates/trie/db/src/storage.rs b/crates/trie/db/src/storage.rs index e249bfb0fd6..55ae3dbbbcb 100644 --- a/crates/trie/db/src/storage.rs +++ b/crates/trie/db/src/storage.rs @@ -114,7 +114,11 @@ mod tests { use super::*; use alloy_consensus::Header; use alloy_primitives::U256; - use reth_db_api::{models::BlockNumberAddress, tables, transaction::DbTxMut}; + use reth_db_api::{ + models::{BlockNumberAddress, StorageBeforeTx, StorageSettings}, + tables, + transaction::DbTxMut, + }; use reth_primitives_traits::StorageEntry; use reth_provider::{ test_utils::create_test_provider_factory, StaticFileProviderFactory, StaticFileSegment, @@ -198,8 +202,6 @@ mod tests { #[test] fn test_hashed_storage_from_reverts_hashed_state() { - use reth_db_api::models::{StorageBeforeTx, StorageSettings}; - let factory = create_test_provider_factory(); factory.set_storage_settings_cache(StorageSettings::v2()); diff --git a/crates/trie/db/src/trie_cursor.rs b/crates/trie/db/src/trie_cursor.rs index cd9c8adc424..f8f0c13faac 100644 --- a/crates/trie/db/src/trie_cursor.rs +++ b/crates/trie/db/src/trie_cursor.rs @@ -369,6 +369,7 @@ mod tests { use alloy_primitives::hex_literal::hex; use reth_db_api::{cursor::DbCursorRW, transaction::DbTxMut}; use reth_provider::test_utils::create_test_provider_factory; + use reth_storage_api::StorageSettingsCache; #[test] fn test_account_trie_order() { @@ -413,7 +414,6 @@ mod tests { // tests that upsert and seek match on the storage trie cursor #[test] fn test_storage_cursor_abstraction() { - use reth_storage_api::StorageSettingsCache; use reth_trie::trie_cursor::{TrieCursor, TrieCursorFactory}; let factory = create_test_provider_factory(); diff --git a/crates/trie/db/src/verify.rs b/crates/trie/db/src/verify.rs new file mode 100644 index 00000000000..6606774f484 --- /dev/null +++ b/crates/trie/db/src/verify.rs @@ -0,0 +1,144 @@ +//! Database-backed trie verification. + +use crate::{ + DatabaseAccountTrieCursor, DatabaseHashedAccountCursor, DatabaseHashedCursorFactory, + DatabaseTrieCursorFactory, TrieTableAdapter, +}; +use alloy_primitives::B256; +use reth_db_api::{ + cursor::{DbCursorRO, DbDupCursorRO}, + database::Database, + tables, + transaction::{CursorTy, DbTx, DupCursorTy}, + DatabaseError, +}; +use reth_execution_errors::StateRootError; +use reth_trie::{ + hashed_cursor::HashedCursorFactory, + trie_cursor::TrieCursorFactory, + verify::{Output, StorageJob, StorageWorker, Verifier}, +}; +use std::fmt; + +/// [`Verifier`] reading database tables through cursors of a `TX` transaction. +type TxVerifier = Verifier< + DatabaseAccountTrieCursor::AccountTrieTable>, A>, + DatabaseHashedAccountCursor>, +>; + +/// Checks the trie tables against the hashed state tables of a database, yielding an [`Output`] +/// for every inconsistency found. See [`Verifier`] for details. +pub struct DatabaseVerifier { + verifier: TxVerifier, + // Backs the cursors held by `verifier`, so it's declared, and dropped, after it. + _txs: [TX; 2], +} + +impl DatabaseVerifier +where + TX: DbTx + 'static, + A: TrieTableAdapter, +{ + /// Starts verification over read transactions opened through `db`. + /// + /// Prevent commits while this constructor opens the transactions so they read the same database + /// version. Each storage worker owns a separate transaction. For long-running verification, + /// configure `db` to disable long-lived read transaction safety. `storage_lookahead` bounds + /// storage jobs whose requested roots have not yet been consumed, including storage without a + /// hashed account row. + /// + /// # Panics + /// + /// Panics if `storage_workers` or `storage_lookahead` is zero. + pub fn new( + db: &impl Database, + storage_workers: usize, + storage_lookahead: usize, + ) -> Result { + assert!(storage_workers > 0, "at least one storage worker is required"); + let account_tx = db.tx()?; + let lookahead_tx = db.tx()?; + let storage_workers = (0..storage_workers) + .map(|_| { + let tx = db.tx()?; + Ok(StorageWorker { + trie: DatabaseTrieCursorFactory::<_, A>::new(&tx) + .storage_trie_cursor(B256::ZERO)?, + hashed: DatabaseHashedCursorFactory::new(&tx) + .hashed_storage_cursor(B256::ZERO)?, + tx, + }) + }) + .collect::, DatabaseError>>()?; + let verifier = Verifier::new( + DatabaseTrieCursorFactory::<_, A>::new(&account_tx).account_trie_cursor()?, + DatabaseHashedCursorFactory::new(&account_tx).hashed_account_cursor()?, + StorageAddresses:: { + hashed: lookahead_tx.cursor_dup_read::()?, + trie: lookahead_tx.cursor_dup_read::()?, + next_hashed: None, + next_trie: None, + initialized: false, + }, + storage_workers, + storage_lookahead, + )?; + Ok(Self { verifier, _txs: [account_tx, lookahead_tx] }) + } +} + +impl Iterator for DatabaseVerifier { + type Item = Result; + + fn next(&mut self) -> Option { + self.verifier.next() + } +} + +impl fmt::Debug for DatabaseVerifier { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("DatabaseVerifier").finish_non_exhaustive() + } +} + +/// Merges the distinct addresses in hashed storage and stored storage tries, including orphan +/// tries. +struct StorageAddresses { + hashed: DupCursorTy, + trie: DupCursorTy, + next_hashed: Option, + next_trie: Option, + initialized: bool, +} + +impl StorageAddresses { + fn next_address(&mut self) -> Result, DatabaseError> { + if !self.initialized { + self.next_hashed = self.hashed.first()?.map(|(address, _)| address); + self.next_trie = self.trie.first()?.map(|(address, _)| address); + self.initialized = true; + } + let address = match (self.next_hashed, self.next_trie) { + (Some(hashed), Some(trie)) => hashed.min(trie), + (Some(address), None) | (None, Some(address)) => address, + (None, None) => return Ok(None), + }; + // Request roots for hashed storage; the account worker discards orphan roots. + let has_storage = self.next_hashed == Some(address); + if has_storage { + self.next_hashed = self.hashed.next_no_dup()?.map(|(address, _)| address); + } + if self.next_trie == Some(address) { + self.next_trie = self.trie.next_no_dup()?.map(|(address, _)| address); + } + Ok(Some(StorageJob { address, root_required: has_storage })) + } +} + +impl Iterator for StorageAddresses { + type Item = Result; + + fn next(&mut self) -> Option { + self.next_address().transpose() + } +} diff --git a/crates/trie/sparse/Cargo.toml b/crates/trie/sparse/Cargo.toml index efc6ea8c2ed..1b4ecdff977 100644 --- a/crates/trie/sparse/Cargo.toml +++ b/crates/trie/sparse/Cargo.toml @@ -46,7 +46,7 @@ proptest.workspace = true rand.workspace = true [features] -dst = [] +fuzz = [] default = ["std", "metrics"] std = [ "dep:either", diff --git a/crates/trie/sparse/src/arena/mod.rs b/crates/trie/sparse/src/arena/mod.rs index 3f336693ee8..d4f04307cc1 100644 --- a/crates/trie/sparse/src/arena/mod.rs +++ b/crates/trie/sparse/src/arena/mod.rs @@ -15,6 +15,7 @@ use alloc::{borrow::Cow, boxed::Box, vec::Vec}; use alloy_primitives::{keccak256, map::B256Map, B256}; use alloy_trie::TrieMask; use core::{cmp::Reverse, mem}; +use rayon::iter::{IntoParallelIterator, IntoParallelRefMutIterator, ParallelIterator}; use reth_execution_errors::{SparseTrieErrorKind, SparseTrieResult}; use reth_trie_common::{ BranchNodeMasks, BranchNodeRef, ExtensionNodeRef, LeafNodeRef, Nibbles, ProofTrieNodeV2, @@ -157,7 +158,7 @@ impl ArenaSparseSubtrie { } /// Asserts that `num_leaves` and `num_dirty_leaves` match the actual counts in the arena. - #[cfg(all(debug_assertions, not(feature = "dst")))] + #[cfg(all(debug_assertions, not(feature = "fuzz")))] fn debug_assert_counters(&self) { let (actual_leaves, actual_dirty) = ArenaParallelSparseTrie::count_leaves_and_dirty(&self.arena, self.root); @@ -273,7 +274,7 @@ impl ArenaSparseSubtrie { self.arena = new_arena; self.root = new_root; - #[cfg(all(debug_assertions, not(feature = "dst")))] + #[cfg(all(debug_assertions, not(feature = "fuzz")))] self.debug_assert_counters(); return pruned; @@ -408,7 +409,7 @@ impl ArenaSparseSubtrie { // Drain remaining cursor entries, propagating dirty state. self.buffers.cursor.drain(&mut self.arena); - #[cfg(all(debug_assertions, not(feature = "dst")))] + #[cfg(all(debug_assertions, not(feature = "fuzz")))] self.debug_assert_counters(); } @@ -444,7 +445,7 @@ impl ArenaSparseSubtrie { // Drain remaining cursor entries, propagating dirty state. self.buffers.cursor.drain(&mut self.arena); - #[cfg(all(debug_assertions, not(feature = "dst")))] + #[cfg(all(debug_assertions, not(feature = "fuzz")))] self.debug_assert_counters(); Ok(()) @@ -463,7 +464,7 @@ impl ArenaSparseSubtrie { new_epoch, ); self.num_dirty_leaves = 0; - #[cfg(all(debug_assertions, not(feature = "dst")))] + #[cfg(all(debug_assertions, not(feature = "fuzz")))] self.debug_assert_counters(); } } @@ -661,7 +662,7 @@ impl ArenaParallelSparseTrie { let (leaves, dirty) = Self::count_leaves_and_dirty(&subtrie.arena, subtrie.root); subtrie.num_leaves = leaves; subtrie.num_dirty_leaves = dirty; - #[cfg(all(debug_assertions, not(feature = "dst")))] + #[cfg(all(debug_assertions, not(feature = "fuzz")))] subtrie.debug_assert_counters(); self.upper_arena[child_idx] = ArenaSparseNode::Subtrie(subtrie); } @@ -1846,7 +1847,7 @@ impl ArenaParallelSparseTrie { /// /// Uses the cursor to DFS the upper arena, checking each visited node's path length. #[instrument(level = "trace", target = TRACE_TARGET, skip_all)] - #[cfg(all(debug_assertions, not(feature = "dst")))] + #[cfg(all(debug_assertions, not(feature = "fuzz")))] fn debug_assert_subtrie_structure(&mut self) { let mut cursor = mem::take(&mut self.buffers.cursor); cursor.reset(&self.upper_arena, self.root, Nibbles::default()); @@ -2017,7 +2018,7 @@ impl ArenaParallelSparseTrie { Some(child_idx) } - #[cfg(all(debug_assertions, not(feature = "dst")))] + #[cfg(all(debug_assertions, not(feature = "fuzz")))] fn collect_reachable_nodes( arena: &NodeArena, idx: Index, @@ -2035,7 +2036,7 @@ impl ArenaParallelSparseTrie { } } - #[cfg(all(debug_assertions, not(feature = "dst")))] + #[cfg(all(debug_assertions, not(feature = "fuzz")))] fn assert_no_orphaned_nodes(arena: &NodeArena, root: Index, label: &str) { let mut reachable = alloy_primitives::map::HashSet::default(); Self::collect_reachable_nodes(arena, root, &mut reachable); @@ -2050,7 +2051,7 @@ impl ArenaParallelSparseTrie { } } -#[cfg(all(debug_assertions, not(feature = "dst")))] +#[cfg(all(debug_assertions, not(feature = "fuzz")))] impl Drop for ArenaParallelSparseTrie { fn drop(&mut self) { Self::assert_no_orphaned_nodes(&self.upper_arena, self.root, "upper arena"); @@ -2267,8 +2268,6 @@ impl SparseTrie for ArenaParallelSparseTrie { let (_, subtrie, node_vec) = &mut taken[0]; subtrie.reveal_nodes(node_vec)?; } else { - use rayon::iter::{IntoParallelRefMutIterator, ParallelIterator}; - let parent_span = tracing::Span::current(); let results: Vec> = taken .par_iter_mut() @@ -2292,7 +2291,7 @@ impl SparseTrie for ArenaParallelSparseTrie { self.restore_taken_subtrie(idx, subtrie); } - #[cfg(all(debug_assertions, not(feature = "dst")))] + #[cfg(all(debug_assertions, not(feature = "fuzz")))] self.debug_assert_subtrie_structure(); Ok(()) @@ -2359,8 +2358,6 @@ impl SparseTrie for ArenaParallelSparseTrie { subtrie.update_cached_rlp(new_epoch); } } else { - use rayon::iter::{IntoParallelIterator, ParallelIterator}; - let parent_span = tracing::Span::current(); taken = taken .into_par_iter() @@ -2559,8 +2556,6 @@ impl SparseTrie for ArenaParallelSparseTrie { let (_, ref mut subtrie) = taken[0]; pruned += subtrie.prune(prune_before); } else { - use rayon::iter::{IntoParallelRefMutIterator, ParallelIterator}; - let parent_span = tracing::Span::current(); pruned += taken .par_iter_mut() @@ -2805,7 +2800,7 @@ impl SparseTrie for ArenaParallelSparseTrie { self.buffers.cursor = cursor; if taken.is_empty() { - #[cfg(all(debug_assertions, not(feature = "dst")))] + #[cfg(all(debug_assertions, not(feature = "fuzz")))] self.debug_assert_subtrie_structure(); return Ok(()); @@ -2816,8 +2811,6 @@ impl SparseTrie for ArenaParallelSparseTrie { let (_, ref mut subtrie, ref range) = taken[0]; subtrie.update_leaves(&sorted[range.clone()]); } else { - use rayon::iter::{IntoParallelRefMutIterator, ParallelIterator}; - let parent_span = tracing::Span::current(); taken.par_iter_mut().for_each(|(_, subtrie, range)| { let _guard = parent_span.enter(); @@ -2882,7 +2875,7 @@ impl SparseTrie for ArenaParallelSparseTrie { self.buffers.cursor = cursor; } - #[cfg(all(debug_assertions, not(feature = "dst")))] + #[cfg(all(debug_assertions, not(feature = "fuzz")))] self.debug_assert_subtrie_structure(); Ok(()) diff --git a/crates/trie/sparse/src/arena/nodes.rs b/crates/trie/sparse/src/arena/nodes.rs index 00eaba72eb3..0b4ccf9210b 100644 --- a/crates/trie/sparse/src/arena/nodes.rs +++ b/crates/trie/sparse/src/arena/nodes.rs @@ -235,7 +235,7 @@ impl ArenaSparseNode { } /// Returns a reference to the subtrie if this is a `Subtrie` node, or `None`. - #[cfg(all(debug_assertions, not(feature = "dst")))] + #[cfg(all(debug_assertions, not(feature = "fuzz")))] pub(super) const fn as_subtrie(&self) -> Option<&ArenaSparseSubtrie> { match self { Self::Subtrie(s) => Some(s), diff --git a/crates/trie/sparse/src/metrics.rs b/crates/trie/sparse/src/metrics.rs index 4e515c48de4..d99c9aa7ecc 100644 --- a/crates/trie/sparse/src/metrics.rs +++ b/crates/trie/sparse/src/metrics.rs @@ -1,5 +1,6 @@ //! Metrics for the sparse state trie +use core::mem::take; use reth_metrics::{metrics::Histogram, Metrics}; /// Metrics for the sparse state trie @@ -16,7 +17,6 @@ pub(crate) struct SparseStateTrieMetrics { impl SparseStateTrieMetrics { /// Record the metrics into the histograms pub(crate) fn record(&mut self) { - use core::mem::take; self.histograms .multiproof_total_account_nodes .record(take(&mut self.multiproof_total_account_nodes) as f64); diff --git a/crates/trie/sparse/src/state.rs b/crates/trie/sparse/src/state.rs index 769d3300d6d..2de76df6def 100644 --- a/crates/trie/sparse/src/state.rs +++ b/crates/trie/sparse/src/state.rs @@ -5,7 +5,13 @@ use crate::{ use alloc::vec::Vec; use alloy_primitives::{map::B256Map, B256}; use either::Either; +#[cfg(feature = "std")] +use rayon::iter::IntoParallelRefMutIterator; +#[cfg(feature = "std")] +use rayon::iter::ParallelIterator; use reth_execution_errors::{SparseStateTrieResult, SparseTrieErrorKind}; +#[cfg(feature = "std")] +use reth_primitives_traits::ParallelBridgeBuffered; use reth_trie_common::{ updates::{StorageTrieUpdates, TrieUpdates}, DecodedMultiProof, MultiProof, Nibbles, ProofTrieNodeV2, @@ -283,9 +289,6 @@ where #[cfg(feature = "std")] let results: Vec<_> = { - use rayon::iter::ParallelIterator; - use reth_primitives_traits::ParallelBridgeBuffered; - let parent_span = tracing::Span::current(); targets .into_iter() @@ -539,8 +542,6 @@ struct StorageTries { impl StorageTries { /// Prunes storage tries by epoch, returning fully old tries to the reuse pool. fn prune(&mut self, prune_before: TrieNodeEpoch, parent_span: &tracing::Span) -> usize { - use rayon::iter::{IntoParallelRefMutIterator, ParallelIterator}; - let addresses_to_evict: Vec = self .tries .par_iter_mut() diff --git a/crates/trie/trie/Cargo.toml b/crates/trie/trie/Cargo.toml index 418ef173f1c..b6eb84da68e 100644 --- a/crates/trie/trie/Cargo.toml +++ b/crates/trie/trie/Cargo.toml @@ -42,6 +42,10 @@ metrics = { workspace = true, optional = true } # `test-utils` feature triehash = { workspace = true, optional = true } +[target.'cfg(not(target_family = "wasm"))'.dependencies] +reth-tasks = { workspace = true, features = ["rayon"] } +crossbeam-channel.workspace = true + [dev-dependencies] # reth reth-primitives-traits = { workspace = true, features = ["test-utils"] } @@ -60,7 +64,7 @@ proptest.workspace = true rand.workspace = true [features] -dst = ["reth-trie-sparse/dst"] +fuzz = ["reth-trie-sparse/fuzz"] metrics = ["reth-metrics", "dep:metrics"] serde = [ "alloy-primitives/serde", @@ -75,6 +79,7 @@ serde = [ ] test-utils = [ "triehash", + "reth-tasks/test-utils", "reth-primitives-traits/test-utils", "reth-trie-common/test-utils", "reth-trie-sparse/test-utils", diff --git a/crates/trie/trie/benches/proof_v2.rs b/crates/trie/trie/benches/proof_v2.rs index 243c5659ae5..e0bf8c12e04 100644 --- a/crates/trie/trie/benches/proof_v2.rs +++ b/crates/trie/trie/benches/proof_v2.rs @@ -10,6 +10,8 @@ use reth_trie::{ proof::StorageProof, proof_v2::StorageProofCalculator, trie_cursor::{mock::MockTrieCursorFactory, TrieCursorFactory}, + updates::StorageTrieUpdates, + StateRoot, }; use reth_trie_common::{HashedPostState, HashedStorage}; @@ -82,8 +84,6 @@ fn generate_test_data( fn create_cursor_factories( post_state: &HashedPostState, ) -> (MockTrieCursorFactory, MockHashedCursorFactory) { - use reth_trie::{updates::StorageTrieUpdates, StateRoot}; - // Create empty trie cursor factory to serve as the initial state for StateRoot // Ensure that there's a storage trie dataset for every storage account let storage_tries: B256Map<_> = post_state diff --git a/crates/trie/trie/src/lib.rs b/crates/trie/trie/src/lib.rs index ee58d94a087..aef0f627032 100644 --- a/crates/trie/trie/src/lib.rs +++ b/crates/trie/trie/src/lib.rs @@ -70,4 +70,5 @@ pub mod test_utils; pub mod mock; /// Verification of existing stored trie nodes against state data. +#[cfg(not(target_family = "wasm"))] pub mod verify; diff --git a/crates/trie/trie/src/node_iter.rs b/crates/trie/trie/src/node_iter.rs index cc5c91f7674..8f4e3d553c6 100644 --- a/crates/trie/trie/src/node_iter.rs +++ b/crates/trie/trie/src/node_iter.rs @@ -277,12 +277,10 @@ where // `can_skip_node` is already set when the hash flag is set, so we don't need // to check for the hash flag explicitly. // - // It is possible that the branch node at the key `seek_key` is not stored in - // the database, so the walker will advance to the branch node after it. Because - // of this, we need to check that the current walker key has a prefix of the key - // that we seeked to. + // A deeper stored key may leave unvisited hashed entries between the seek + // prefix and the walker, so only skip the seek at the exact prefix. if can_skip_node && - self.walker.key().is_some_and(|key| key.starts_with(&seek_prefix)) && + self.walker.key().is_some_and(|key| key == &seek_prefix) && self.walker.children_are_in_trie() { trace!( @@ -307,22 +305,26 @@ where #[cfg(test)] mod tests { - use super::{TrieElement, TrieNodeIter}; + use super::*; use crate::{ hashed_cursor::{ - mock::MockHashedCursorFactory, noop::NoopHashedCursor, HashedCursorFactory, - HashedPostStateCursor, + mock::MockHashedCursorFactory, + noop::{NoopHashedCursor, NoopHashedCursorFactory}, + HashedCursorFactory, HashedPostStateCursor, HashedPostStateCursorFactory, }, mock::{KeyVisit, KeyVisitType}, trie_cursor::{ - mock::MockTrieCursorFactory, noop::NoopAccountTrieCursor, TrieCursorFactory, + mock::MockTrieCursorFactory, + noop::{NoopAccountTrieCursor, NoopTrieCursorFactory}, + InMemoryTrieCursorFactory, TrieCursorFactory, }, walker::TrieWalker, + StateRoot, }; use alloy_primitives::{ b256, map::{B256Map, HashMap}, - B256, + B256, U256, }; use alloy_trie::{ BranchNodeCompact, HashBuilder, Nibbles, TrieAccount, TrieMask, EMPTY_ROOT_HASH, @@ -330,8 +332,8 @@ mod tests { use itertools::Itertools; use reth_primitives_traits::Account; use reth_trie_common::{ - prefix_set::PrefixSetMut, updates::TrieUpdates, BranchNode, HashedPostState, LeafNode, - RlpNode, + prefix_set::PrefixSetMut, updates::TrieUpdates, BranchNode, HashedPostState, HashedStorage, + LeafNode, RlpNode, }; use std::collections::BTreeMap; @@ -382,6 +384,71 @@ mod tests { trie_updates.account_nodes } + #[test] + fn inserted_leaf_before_deeper_stored_branch_is_included() { + fn slot(prefix: &[u8]) -> B256 { + let mut bytes = [0; 32]; + bytes[..prefix.len()].copy_from_slice(prefix); + B256::from(bytes) + } + + let address = B256::repeat_byte(0xaa); + let account = || Account { nonce: 1, ..Default::default() }; + let old_slots = [ + slot(&[0x00]), + slot(&[0x0f, 0x00]), + slot(&[0x0f, 0x10]), + slot(&[0x10, 0xa0, 0x00]), + slot(&[0x10, 0xa0, 0x01]), + slot(&[0x10, 0xa0, 0x10]), + slot(&[0x10, 0xa1]), + ]; + let inserted = slot(&[0x10, 0x00]); + let old_state = + HashedPostState::default().with_accounts([(address, Some(account()))]).with_storages([ + (address, HashedStorage::from_iter(old_slots.map(|slot| (slot, U256::from(1))))), + ]); + let old_sorted = old_state.clone().into_sorted(); + let (old_root, old_updates) = StateRoot::new( + NoopTrieCursorFactory::default(), + HashedPostStateCursorFactory::new(NoopHashedCursorFactory::default(), &old_sorted), + ) + .with_prefix_sets(old_state.construct_prefix_sets().freeze()) + .root_with_updates() + .unwrap(); + + let new_state = HashedPostState::default() + .with_accounts([(address, Some(account()))]) + .with_storages([( + address, + HashedStorage::from_iter( + old_slots.into_iter().chain([inserted]).map(|slot| (slot, U256::from(1))), + ), + )]); + let delta = HashedPostState::default() + .with_accounts([(address, Some(account()))]) + .with_storages([(address, HashedStorage::from_iter([(inserted, U256::from(1))]))]); + let new_sorted = new_state.clone().into_sorted(); + let rebuilt_root = StateRoot::new( + NoopTrieCursorFactory::default(), + HashedPostStateCursorFactory::new(NoopHashedCursorFactory::default(), &new_sorted), + ) + .with_prefix_sets(new_state.construct_prefix_sets().freeze()) + .root() + .unwrap(); + let old_trie = old_updates.into_sorted(); + let incremental_root = StateRoot::new( + InMemoryTrieCursorFactory::new(NoopTrieCursorFactory::default(), &old_trie), + HashedPostStateCursorFactory::new(NoopHashedCursorFactory::default(), &new_sorted), + ) + .with_prefix_sets(delta.construct_prefix_sets().freeze()) + .root() + .unwrap(); + + assert_ne!(old_root, rebuilt_root); + assert_eq!(incremental_root, rebuilt_root); + } + #[test] fn test_trie_node_iter() { fn empty_leaf_rlp_for_key(key: Nibbles) -> RlpNode { diff --git a/crates/trie/trie/src/trie.rs b/crates/trie/trie/src/trie.rs index 40512b90f5d..03b23a31b81 100644 --- a/crates/trie/trie/src/trie.rs +++ b/crates/trie/trie/src/trie.rs @@ -22,7 +22,7 @@ use tracing::{debug, instrument, trace, Span}; /// The default updates after which root algorithms should return intermediate progress rather than /// finishing the computation. -const DEFAULT_INTERMEDIATE_THRESHOLD: u64 = 100_000; +pub(crate) const DEFAULT_INTERMEDIATE_THRESHOLD: u64 = 100_000; #[cfg(feature = "metrics")] use crate::metrics::{StateRootMetrics, TrieRootMetrics}; @@ -228,20 +228,20 @@ where Some(self.trie_cursor_factory.storage_trie_cursor(hashed_address)?); } - let storage_result = StorageRoot::::calculate_with_cursors( - StorageRootCalculation { - hashed_address, - prefix_set: self - .prefix_sets - .storage_prefix_sets - .get(&hashed_address) - .cloned() - .unwrap_or_default(), - previous_state: Some(storage_state.state), - walk_all_changed_branch_children: self.walk_all_changed_branch_children, - threshold: remaining_threshold, - retain_updates, - }, + let storage_result = StorageRootCalculation { + hashed_address, + prefix_set: self + .prefix_sets + .storage_prefix_sets + .get(&hashed_address) + .cloned() + .unwrap_or_default(), + previous_state: Some(storage_state.state), + walk_all_changed_branch_children: self.walk_all_changed_branch_children, + threshold: remaining_threshold, + retain_updates, + } + .calculate( storage_trie_cursor.as_mut().expect("storage trie cursor is initialized"), hashed_storage_cursor.as_mut().expect("hashed storage cursor is initialized"), #[cfg(feature = "metrics")] @@ -301,20 +301,20 @@ where Some(self.trie_cursor_factory.storage_trie_cursor(hashed_address)?); } - let storage_result = StorageRoot::::calculate_with_cursors( - StorageRootCalculation { - hashed_address, - prefix_set: self - .prefix_sets - .storage_prefix_sets - .get(&hashed_address) - .cloned() - .unwrap_or_default(), - previous_state: None, - walk_all_changed_branch_children: self.walk_all_changed_branch_children, - threshold: remaining_threshold, - retain_updates, - }, + let storage_result = StorageRootCalculation { + hashed_address, + prefix_set: self + .prefix_sets + .storage_prefix_sets + .get(&hashed_address) + .cloned() + .unwrap_or_default(), + previous_state: None, + walk_all_changed_branch_children: self.walk_all_changed_branch_children, + threshold: remaining_threshold, + retain_updates, + } + .calculate( storage_trie_cursor.as_mut().expect("storage trie cursor is initialized"), hashed_storage_cursor .as_mut() @@ -711,15 +711,15 @@ where hashed_cursor_factory.hashed_storage_cursor(hashed_address)?; let mut storage_trie_cursor = trie_cursor_factory.storage_trie_cursor(hashed_address)?; - Self::calculate_with_cursors( - StorageRootCalculation { - hashed_address, - prefix_set, - previous_state, - walk_all_changed_branch_children, - threshold, - retain_updates, - }, + StorageRootCalculation { + hashed_address, + prefix_set, + previous_state, + walk_all_changed_branch_children, + threshold, + retain_updates, + } + .calculate( &mut storage_trie_cursor, &mut hashed_storage_cursor, #[cfg(feature = "metrics")] @@ -754,26 +754,38 @@ where let mut storage_trie_cursor = trie_cursor_factory.storage_trie_cursor(hashed_address)?; - Self::calculate_with_cursors( - StorageRootCalculation { - hashed_address, - prefix_set, - previous_state, - walk_all_changed_branch_children, - threshold, - retain_updates, - }, + StorageRootCalculation { + hashed_address, + prefix_set, + previous_state, + walk_all_changed_branch_children, + threshold, + retain_updates, + } + .calculate( &mut storage_trie_cursor, hashed_storage_cursor, #[cfg(feature = "metrics")] &metrics, ) } +} + +/// Parameters for a storage root calculation using pre-created cursors. +pub(crate) struct StorageRootCalculation { + pub(crate) hashed_address: B256, + pub(crate) prefix_set: PrefixSet, + pub(crate) previous_state: Option, + pub(crate) walk_all_changed_branch_children: bool, + pub(crate) threshold: u64, + pub(crate) retain_updates: bool, +} +impl StorageRootCalculation { /// Walks the hashed storage table entries for a given address and calculates the storage root /// using pre-created cursors. The cursors will be repositioned to the given hashed address. - fn calculate_with_cursors( - calculation: StorageRootCalculation, + pub(crate) fn calculate( + self, trie_cursor: &mut TC, hashed_storage_cursor: &mut HC, #[cfg(feature = "metrics")] metrics: &TrieRootMetrics, @@ -782,14 +794,14 @@ where TC: TrieStorageCursor, HC: HashedStorageCursor, { - let StorageRootCalculation { + let Self { hashed_address, prefix_set, previous_state, walk_all_changed_branch_children, threshold, retain_updates, - } = calculation; + } = self; hashed_storage_cursor.set_hashed_address(hashed_address); // Empty storage only needs to be walked when changed prefixes must produce trie updates. @@ -900,16 +912,6 @@ where } } -/// Parameters for a storage root calculation using pre-created cursors. -struct StorageRootCalculation { - hashed_address: B256, - prefix_set: PrefixSet, - previous_state: Option, - walk_all_changed_branch_children: bool, - threshold: u64, - retain_updates: bool, -} - /// Trie type for differentiating between various trie calculations. #[derive(Clone, Copy, Debug)] pub enum TrieType { diff --git a/crates/trie/trie/src/trie_cursor/in_memory.rs b/crates/trie/trie/src/trie_cursor/in_memory.rs index 8db7853e804..b31e0d06ff7 100644 --- a/crates/trie/trie/src/trie_cursor/in_memory.rs +++ b/crates/trie/trie/src/trie_cursor/in_memory.rs @@ -393,6 +393,7 @@ mod tests { use crate::trie_cursor::mock::MockTrieCursor; use parking_lot::Mutex; use std::{collections::BTreeMap, sync::Arc}; + use tracing::debug; #[derive(Debug)] struct InMemoryTrieCursorTestCase { @@ -801,7 +802,6 @@ mod tests { #[test] fn test_all_storage_nodes_deleted_exact_keys() { - use tracing::debug; reth_tracing::init_test_tracing(); // This test reproduces an edge case where: diff --git a/crates/trie/trie/src/verify.rs b/crates/trie/trie/src/verify.rs deleted file mode 100644 index 5ab0c848625..00000000000 --- a/crates/trie/trie/src/verify.rs +++ /dev/null @@ -1,1018 +0,0 @@ -use crate::{ - hashed_cursor::{HashedCursor, HashedCursorFactory}, - progress::{IntermediateStateRootState, StateRootProgress}, - trie::StateRoot, - trie_cursor::{ - depth_first::{self, DepthFirstTrieIterator}, - noop::NoopTrieCursorFactory, - TrieCursor, TrieCursorFactory, - }, - Nibbles, -}; -use alloy_primitives::B256; -use alloy_trie::BranchNodeCompact; -use reth_execution_errors::StateRootError; -use reth_storage_errors::db::DatabaseError; -use std::cmp::{Ordering, Reverse}; -use tracing::trace; - -/// Used by [`StateRootBranchNodesIter`] to iterate over branch nodes in a state root. -#[derive(Debug)] -enum BranchNode { - Account(Nibbles, BranchNodeCompact), - Storage(B256, Nibbles, BranchNodeCompact), -} - -/// Iterates over branch nodes produced by a [`StateRoot`]. The `StateRoot` will only used the -/// hashed accounts/storages tables, meaning it is recomputing the trie from scratch without the use -/// of the trie tables. -/// -/// [`BranchNode`]s are iterated over such that: -/// * Account nodes and storage nodes may be interleaved. -/// * Storage nodes for the same account will be ordered by ascending path relative to each other. -/// * Account nodes will be ordered by ascending path relative to each other. -/// * All storage nodes for one account will finish before storage nodes for another account are -/// started. In other words, if the current storage account is not equal to the previous, the -/// previous has no more nodes. -#[derive(Debug)] -struct StateRootBranchNodesIter { - hashed_cursor_factory: H, - account_nodes: Vec<(Nibbles, BranchNodeCompact)>, - storage_tries: Vec<(B256, Vec<(Nibbles, BranchNodeCompact)>)>, - curr_storage: Option<(B256, Vec<(Nibbles, BranchNodeCompact)>)>, - intermediate_state: Option>, - complete: bool, -} - -impl StateRootBranchNodesIter { - fn new(hashed_cursor_factory: H) -> Self { - Self { - hashed_cursor_factory, - account_nodes: Default::default(), - storage_tries: Default::default(), - curr_storage: None, - intermediate_state: None, - complete: false, - } - } - - /// Sorts a Vec of updates such that it is ready to be yielded from the `next` method. We yield - /// by popping off of the account/storage vecs, so we sort them in reverse order. - /// - /// Depth-first sorting is used because this is the order that the `HashBuilder` computes - /// branch nodes internally, even if it produces them as `B256Map`s. - fn sort_updates(updates: &mut [(Nibbles, BranchNodeCompact)]) { - updates.sort_unstable_by(|a, b| depth_first::cmp(&b.0, &a.0)); - } -} - -impl Iterator for StateRootBranchNodesIter { - type Item = Result; - - fn next(&mut self) -> Option { - loop { - // If we already started iterating through a storage trie's updates, continue doing - // so. - if let Some((account, storage_updates)) = self.curr_storage.as_mut() && - let Some((path, node)) = storage_updates.pop() - { - let node = BranchNode::Storage(*account, path, node); - return Some(Ok(node)) - } - - // If there's not a storage trie already being iterated over then check if there's a - // storage trie we could start iterating over. - if let Some((account, storage_updates)) = self.storage_tries.pop() { - debug_assert!(!storage_updates.is_empty()); - - self.curr_storage = Some((account, storage_updates)); - continue; - } - - // `storage_updates` is empty, check if there are account updates. - if let Some((path, node)) = self.account_nodes.pop() { - return Some(Ok(BranchNode::Account(path, node))) - } - - // All data from any previous runs of the `StateRoot` has been produced, run the next - // partial computation, unless `StateRootProgress::Complete` has been returned in which - // case iteration is over. - if self.complete { - return None - } - - let state_root = - StateRoot::new(NoopTrieCursorFactory, self.hashed_cursor_factory.clone()) - .with_intermediate_state(self.intermediate_state.take().map(|s| *s)); - - let updates = match state_root.root_with_progress() { - Err(err) => return Some(Err(err)), - Ok(StateRootProgress::Complete(_, _, updates)) => { - self.complete = true; - updates - } - Ok(StateRootProgress::Progress(intermediate_state, _, updates)) => { - self.intermediate_state = Some(intermediate_state); - updates - } - }; - - // collect account updates and sort them in descending order, so that when we pop them - // off the Vec they are popped in ascending order. - self.account_nodes.extend(updates.account_nodes); - Self::sort_updates(&mut self.account_nodes); - - self.storage_tries = updates - .storage_tries - .into_iter() - .filter_map(|(account, t)| { - (!t.storage_nodes.is_empty()).then(|| { - let mut storage_nodes = t.storage_nodes.into_iter().collect::>(); - Self::sort_updates(&mut storage_nodes); - (account, storage_nodes) - }) - }) - .collect::>(); - - // `root_with_progress` will output storage updates ordered by their account hash. If - // `root_with_progress` only returns a partial result then it will pick up where - // it left off in the storage trie on the next run. - // - // By sorting by the account we ensure that we continue with the partially processed - // trie (the last of the previous run) first. We sort in reverse order because we pop - // off of this Vec. - self.storage_tries.sort_unstable_by_key(|a| Reverse(a.0)); - - // loop back to the top. - } - } -} - -/// Output describes an inconsistency found when comparing the hashed state tables -/// ([`HashedCursorFactory`]) with that of the trie tables ([`TrieCursorFactory`]). The hashed -/// tables are considered the source of truth; outputs are on the part of the trie tables. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum Output { - /// An extra account node was found. - AccountExtra(Nibbles, BranchNodeCompact), - /// An extra storage node was found. - StorageExtra(B256, Nibbles, BranchNodeCompact), - /// An account node had the wrong value. - AccountWrong { - /// Path of the node - path: Nibbles, - /// The node's expected value. - expected: BranchNodeCompact, - /// The node's found value. - found: BranchNodeCompact, - }, - /// A storage node had the wrong value. - StorageWrong { - /// The account the storage trie belongs to. - account: B256, - /// Path of the node - path: Nibbles, - /// The node's expected value. - expected: BranchNodeCompact, - /// The node's found value. - found: BranchNodeCompact, - }, - /// An account node was missing. - AccountMissing(Nibbles, BranchNodeCompact), - /// A storage node was missing. - StorageMissing(B256, Nibbles, BranchNodeCompact), - /// Progress indicator with the last seen account path. - Progress(Nibbles), -} - -/// Verifies the contents of a trie table against some other data source which is able to produce -/// stored trie nodes. -#[derive(Debug)] -struct SingleVerifier { - account: Option, // None for accounts trie - trie_iter: I, - curr: Option<(Nibbles, BranchNodeCompact)>, -} - -impl SingleVerifier> { - fn new(account: Option, trie_cursor: C) -> Result { - let mut trie_iter = DepthFirstTrieIterator::new(trie_cursor); - let curr = trie_iter.next().transpose()?; - Ok(Self { account, trie_iter, curr }) - } - - const fn output_extra(&self, path: Nibbles, node: BranchNodeCompact) -> Output { - if let Some(account) = self.account { - Output::StorageExtra(account, path, node) - } else { - Output::AccountExtra(path, node) - } - } - - const fn output_wrong( - &self, - path: Nibbles, - expected: BranchNodeCompact, - found: BranchNodeCompact, - ) -> Output { - if let Some(account) = self.account { - Output::StorageWrong { account, path, expected, found } - } else { - Output::AccountWrong { path, expected, found } - } - } - - const fn output_missing(&self, path: Nibbles, node: BranchNodeCompact) -> Output { - if let Some(account) = self.account { - Output::StorageMissing(account, path, node) - } else { - Output::AccountMissing(path, node) - } - } - - /// Called with the next path and node in the canonical sequence of stored trie nodes. Will - /// append to the given `outputs` Vec if walking the trie cursor produces data - /// inconsistent with that given. - /// - /// `next` must be called with paths in depth-first order. - fn next( - &mut self, - outputs: &mut Vec, - path: Nibbles, - node: BranchNodeCompact, - ) -> Result<(), DatabaseError> { - loop { - // `curr` is None only if the end of the iterator has been reached. Any further nodes - // found must be considered missing. - if self.curr.is_none() { - outputs.push(self.output_missing(path, node)); - return Ok(()) - } - - let (curr_path, curr_node) = self.curr.as_ref().expect("not None"); - trace!(target: "trie::verify", account=?self.account, ?curr_path, ?path, "Current cursor node"); - - // Use depth-first ordering for comparison - match depth_first::cmp(&path, curr_path) { - Ordering::Less => { - // If the given path comes before the cursor's current path in depth-first - // order, then the given path was not produced by the cursor. - outputs.push(self.output_missing(path, node)); - return Ok(()) - } - Ordering::Equal => { - // If the current path matches the given one (happy path) but the nodes - // aren't equal then we produce a wrong node. Either way we want to move the - // iterator forward. - if *curr_node != node { - outputs.push(self.output_wrong(path, node, curr_node.clone())) - } - self.curr = self.trie_iter.next().transpose()?; - return Ok(()) - } - Ordering::Greater => { - // If the given path comes after the current path in depth-first order, - // it means the cursor's path was not found by the caller (otherwise it would - // have hit the equal case) and so is extraneous. - outputs.push(self.output_extra(*curr_path, curr_node.clone())); - self.curr = self.trie_iter.next().transpose()?; - // back to the top of the loop to check the latest `self.curr` value against the - // given path/node. - } - } - } - } - - /// Must be called once there are no more calls to `next` to made. All further nodes produced - /// by the iterator will be considered extraneous. - fn finalize(&mut self, outputs: &mut Vec) -> Result<(), DatabaseError> { - loop { - if let Some((curr_path, curr_node)) = self.curr.take() { - outputs.push(self.output_extra(curr_path, curr_node)); - self.curr = self.trie_iter.next().transpose()?; - } else { - return Ok(()) - } - } - } -} - -/// Checks that data stored in the trie database is consistent, using hashed accounts/storages -/// database tables as the source of truth. This will iteratively recompute the entire trie based -/// on the hashed state, and produce any discovered [`Output`]s via the `next` method. -#[derive(Debug)] -pub struct Verifier<'a, T: TrieCursorFactory, H> { - trie_cursor_factory: &'a T, - hashed_cursor_factory: H, - branch_node_iter: StateRootBranchNodesIter, - outputs: Vec, - account: SingleVerifier>>, - storage: Option<(B256, SingleVerifier>>)>, - complete: bool, -} - -impl<'a, T: TrieCursorFactory, H: HashedCursorFactory + Clone> Verifier<'a, T, H> { - /// Creates a new verifier instance. - pub fn new( - trie_cursor_factory: &'a T, - hashed_cursor_factory: H, - ) -> Result { - Ok(Self { - trie_cursor_factory, - hashed_cursor_factory: hashed_cursor_factory.clone(), - branch_node_iter: StateRootBranchNodesIter::new(hashed_cursor_factory), - outputs: Default::default(), - account: SingleVerifier::new(None, trie_cursor_factory.account_trie_cursor()?)?, - storage: None, - complete: false, - }) - } -} - -impl<'a, T: TrieCursorFactory, H: HashedCursorFactory + Clone> Verifier<'a, T, H> { - fn new_storage( - &mut self, - account: B256, - path: Nibbles, - node: BranchNodeCompact, - ) -> Result<(), DatabaseError> { - let trie_cursor = self.trie_cursor_factory.storage_trie_cursor(account)?; - let mut storage = SingleVerifier::new(Some(account), trie_cursor)?; - storage.next(&mut self.outputs, path, node)?; - self.storage = Some((account, storage)); - Ok(()) - } - - /// This method is called using the account hashes at the boundary of [`BranchNode::Storage`] - /// sequences, ie once the [`StateRootBranchNodesIter`] has begun yielding storage nodes for a - /// different account than it was yielding previously. All accounts between the two should have - /// empty storages. - fn verify_empty_storages( - &mut self, - last_account: B256, - next_account: B256, - start_inclusive: bool, - end_inclusive: bool, - ) -> Result<(), DatabaseError> { - let mut account_cursor = self.hashed_cursor_factory.hashed_account_cursor()?; - let mut account_seeked = false; - - if !start_inclusive { - account_seeked = true; - account_cursor.seek(last_account)?; - } - - loop { - let Some((curr_account, _)) = (if account_seeked { - account_cursor.next()? - } else { - account_seeked = true; - account_cursor.seek(last_account)? - }) else { - return Ok(()) - }; - - if curr_account < next_account || (end_inclusive && curr_account == next_account) { - trace!(target: "trie::verify", account = ?curr_account, "Verifying account has empty storage"); - - let mut storage_cursor = - self.trie_cursor_factory.storage_trie_cursor(curr_account)?; - let mut seeked = false; - while let Some((path, node)) = if seeked { - storage_cursor.next()? - } else { - seeked = true; - storage_cursor.seek(Nibbles::new())? - } { - self.outputs.push(Output::StorageExtra(curr_account, path, node)); - } - } else { - return Ok(()) - } - } - } - - fn try_next(&mut self) -> Result<(), StateRootError> { - match self.branch_node_iter.next().transpose()? { - None => { - self.account.finalize(&mut self.outputs)?; - if let Some((prev_account, storage)) = self.storage.as_mut() { - storage.finalize(&mut self.outputs)?; - - // If there was a previous storage account, and it is the final one, then we - // need to validate that all accounts coming after it have empty storages. - let prev_account = *prev_account; - - // Calculate the max possible account address (all bits set). - let max_account = B256::repeat_byte(0xFFu8); - - self.verify_empty_storages(prev_account, max_account, false, true)?; - } - self.complete = true; - } - Some(BranchNode::Account(path, node)) => { - trace!(target: "trie::verify", ?path, "Account node from state root"); - self.account.next(&mut self.outputs, path, node)?; - // Push progress indicator - if !path.is_empty() { - self.outputs.push(Output::Progress(path)); - } - } - Some(BranchNode::Storage(account, path, node)) => { - trace!(target: "trie::verify", ?account, ?path, "Storage node from state root"); - match self.storage.as_mut() { - None => { - // First storage account - check for any empty storages before it - self.verify_empty_storages(B256::ZERO, account, true, false)?; - self.new_storage(account, path, node)?; - } - Some((prev_account, storage)) if *prev_account == account => { - storage.next(&mut self.outputs, path, node)?; - } - Some((prev_account, storage)) => { - storage.finalize(&mut self.outputs)?; - // Clear any storage entries between the previous account and the new one - let prev_account = *prev_account; - self.verify_empty_storages(prev_account, account, false, false)?; - self.new_storage(account, path, node)?; - } - } - } - } - - // If any outputs were appended we want to reverse them, so they are popped off - // in the same order they were appended. - self.outputs.reverse(); - Ok(()) - } -} - -impl<'a, T: TrieCursorFactory, H: HashedCursorFactory + Clone> Iterator for Verifier<'a, T, H> { - type Item = Result; - - fn next(&mut self) -> Option { - loop { - if let Some(output) = self.outputs.pop() { - return Some(Ok(output)) - } - - if self.complete { - return None - } - - if let Err(err) = self.try_next() { - return Some(Err(err)) - } - } - } -} - -#[cfg(test)] -mod tests { - - use super::*; - use crate::{ - hashed_cursor::mock::MockHashedCursorFactory, - trie_cursor::mock::{MockTrieCursor, MockTrieCursorFactory}, - }; - use alloy_primitives::{keccak256, map::B256Map, Address, U256}; - use alloy_trie::TrieMask; - use assert_matches::assert_matches; - use reth_primitives_traits::Account; - use std::collections::BTreeMap; - - /// Helper function to create a simple test `BranchNodeCompact` - fn test_branch_node( - state_mask: u16, - tree_mask: u16, - hash_mask: u16, - hashes: Vec, - ) -> BranchNodeCompact { - // Ensure the number of hashes matches the number of bits set in hash_mask - let expected_hashes = hash_mask.count_ones() as usize; - let mut final_hashes = hashes; - let mut counter = 100u8; - while final_hashes.len() < expected_hashes { - final_hashes.push(B256::repeat_byte(counter)); - counter += 1; - } - final_hashes.truncate(expected_hashes); - - BranchNodeCompact::new( - TrieMask::new(state_mask), - TrieMask::new(tree_mask), - TrieMask::new(hash_mask), - final_hashes, - None, - ) - } - - /// Helper function to create a simple test `MockTrieCursor` - fn create_mock_cursor(trie_nodes: BTreeMap) -> MockTrieCursor { - let factory = MockTrieCursorFactory::new(trie_nodes, B256Map::default()); - factory.account_trie_cursor().unwrap() - } - - #[test] - fn test_state_root_branch_nodes_iter_empty() { - // Test with completely empty state - let factory = MockHashedCursorFactory::new(BTreeMap::new(), B256Map::default()); - let mut iter = StateRootBranchNodesIter::new(factory); - - // Collect all results - with empty state, should complete without producing nodes - let mut count = 0; - for result in iter.by_ref() { - assert!(result.is_ok(), "Unexpected error: {:?}", result.unwrap_err()); - count += 1; - // Prevent infinite loop in test - assert!(count <= 1000, "Too many iterations"); - } - - assert!(iter.complete); - } - - #[test] - #[allow(clippy::needless_update)] - fn test_state_root_branch_nodes_iter_basic() { - // Simple test with a few accounts and storage - let mut accounts = BTreeMap::new(); - let mut storage_tries = B256Map::default(); - - // Create test accounts - let addr1 = keccak256(Address::with_last_byte(1)); - accounts.insert( - addr1, - Account { - nonce: 1, - balance: U256::from(1000), - bytecode_hash: Some(keccak256(b"code1")), - ..Default::default() - }, - ); - - // Add storage for the account - let mut storage1 = BTreeMap::new(); - storage1.insert(keccak256(B256::with_last_byte(1)), U256::from(100)); - storage1.insert(keccak256(B256::with_last_byte(2)), U256::from(200)); - storage_tries.insert(addr1, storage1); - - let factory = MockHashedCursorFactory::new(accounts, storage_tries); - let mut iter = StateRootBranchNodesIter::new(factory); - - // Collect nodes and verify basic properties - let mut account_paths = Vec::new(); - let mut storage_paths_by_account: B256Map> = B256Map::default(); - let mut iterations = 0; - - for result in iter.by_ref() { - iterations += 1; - assert!(iterations <= 10000, "Too many iterations - possible infinite loop"); - - match result { - Ok(BranchNode::Account(path, _)) => { - account_paths.push(path); - } - Ok(BranchNode::Storage(account, path, _)) => { - storage_paths_by_account.entry(account).or_default().push(path); - } - Err(e) => panic!("Unexpected error: {:?}", e), - } - } - - // Verify account paths are in ascending order - for i in 1..account_paths.len() { - assert!( - account_paths[i - 1] < account_paths[i], - "Account paths should be in ascending order" - ); - } - - // Verify storage paths for each account are in ascending order - for (account, paths) in storage_paths_by_account { - for i in 1..paths.len() { - assert!( - paths[i - 1] < paths[i], - "Storage paths for account {:?} should be in ascending order", - account - ); - } - } - - assert!(iter.complete); - } - - #[test] - #[allow(clippy::needless_update)] - fn test_state_root_branch_nodes_iter_multiple_accounts() { - // Test with multiple accounts to verify ordering - let mut accounts = BTreeMap::new(); - let mut storage_tries = B256Map::default(); - - // Create multiple test addresses - for i in 1u8..=3 { - let addr = keccak256([i; 20]); - accounts.insert( - addr, - Account { - nonce: i as u64, - balance: U256::from(i as u64 * 1000), - bytecode_hash: (i == 2).then(|| keccak256([i])), - ..Default::default() - }, - ); - - // Add some storage for each account - let mut storage = BTreeMap::new(); - for j in 0..i { - storage.insert(keccak256(B256::with_last_byte(j)), U256::from(j as u64 * 10)); - } - if !storage.is_empty() { - storage_tries.insert(addr, storage); - } - } - - let factory = MockHashedCursorFactory::new(accounts, storage_tries); - let mut iter = StateRootBranchNodesIter::new(factory); - - // Track what we see - let mut seen_storage_accounts = Vec::new(); - let mut current_storage_account = None; - let mut iterations = 0; - - for result in iter.by_ref() { - iterations += 1; - assert!(iterations <= 10000, "Too many iterations"); - - match result { - Ok(BranchNode::Storage(account, _, _)) => { - if current_storage_account != Some(account) { - // Verify we don't revisit a storage account - assert!( - !seen_storage_accounts.contains(&account), - "Should not revisit storage account {:?}", - account - ); - seen_storage_accounts.push(account); - current_storage_account = Some(account); - } - } - Ok(BranchNode::Account(_, _)) => { - // Account nodes are fine - } - Err(e) => panic!("Unexpected error: {:?}", e), - } - } - - assert!(iter.complete); - } - - #[test] - fn test_single_verifier_new() { - // Test creating a new SingleVerifier for account trie - let trie_nodes = BTreeMap::from([( - Nibbles::from_nibbles([0x1]), - test_branch_node(0b1111, 0, 0, vec![]), - )]); - - let cursor = create_mock_cursor(trie_nodes); - let verifier = SingleVerifier::new(None, cursor).unwrap(); - - // Should have seeked to the beginning and found the first node - assert!(verifier.curr.is_some()); - } - - #[test] - fn test_single_verifier_next_exact_match() { - // Test when the expected node matches exactly - let node1 = test_branch_node(0b1111, 0, 0b1111, vec![B256::repeat_byte(1u8)]); - let node2 = test_branch_node(0b0101, 0b0001, 0b0100, vec![B256::repeat_byte(2u8)]); - - let trie_nodes = BTreeMap::from([ - (Nibbles::from_nibbles([0x1]), node1.clone()), - (Nibbles::from_nibbles([0x2]), node2), - ]); - - let cursor = create_mock_cursor(trie_nodes); - let mut verifier = SingleVerifier::new(None, cursor).unwrap(); - let mut outputs = Vec::new(); - - // Call next with the exact node that exists - verifier.next(&mut outputs, Nibbles::from_nibbles([0x1]), node1).unwrap(); - - // Should have no outputs - assert!(outputs.is_empty()); - } - - #[test] - fn test_single_verifier_next_wrong_value() { - // Test when the path matches but value is different - let node_in_trie = test_branch_node(0b1111, 0, 0b1111, vec![B256::repeat_byte(1u8)]); - let node_expected = test_branch_node(0b0101, 0b0001, 0b0100, vec![B256::repeat_byte(2u8)]); - - let trie_nodes = BTreeMap::from([(Nibbles::from_nibbles([0x1]), node_in_trie.clone())]); - - let cursor = create_mock_cursor(trie_nodes); - let mut verifier = SingleVerifier::new(None, cursor).unwrap(); - let mut outputs = Vec::new(); - - // Call next with different node value - verifier.next(&mut outputs, Nibbles::from_nibbles([0x1]), node_expected.clone()).unwrap(); - - // Should have one "wrong" output - assert_eq!(outputs.len(), 1); - assert_matches!( - &outputs[0], - Output::AccountWrong { path, expected, found } - if *path == Nibbles::from_nibbles([0x1]) && *expected == node_expected && *found == node_in_trie - ); - } - - #[test] - fn test_single_verifier_next_missing() { - // Test when expected node doesn't exist in trie - let node1 = test_branch_node(0b1111, 0, 0b1111, vec![B256::repeat_byte(1u8)]); - let node_missing = test_branch_node(0b0101, 0b0001, 0b0100, vec![B256::repeat_byte(2u8)]); - - let trie_nodes = BTreeMap::from([(Nibbles::from_nibbles([0x3]), node1)]); - - let cursor = create_mock_cursor(trie_nodes); - let mut verifier = SingleVerifier::new(None, cursor).unwrap(); - let mut outputs = Vec::new(); - - // Call next with a node that comes before any in the trie - verifier.next(&mut outputs, Nibbles::from_nibbles([0x1]), node_missing.clone()).unwrap(); - - // Should have one "missing" output - assert_eq!(outputs.len(), 1); - assert_matches!( - &outputs[0], - Output::AccountMissing(path, node) - if *path == Nibbles::from_nibbles([0x1]) && *node == node_missing - ); - } - - #[test] - fn test_single_verifier_next_extra() { - // Test when trie has extra nodes not in expected - // Create a proper trie structure with root - let node_root = test_branch_node(0b1110, 0, 0b1110, vec![]); // root has children at 1, 2, 3 - let node1 = test_branch_node(0b0001, 0, 0b0001, vec![]); - let node2 = test_branch_node(0b0010, 0, 0b0010, vec![]); - let node3 = test_branch_node(0b0100, 0, 0b0100, vec![]); - - let trie_nodes = BTreeMap::from([ - (Nibbles::new(), node_root.clone()), - (Nibbles::from_nibbles([0x1]), node1.clone()), - (Nibbles::from_nibbles([0x2]), node2.clone()), - (Nibbles::from_nibbles([0x3]), node3.clone()), - ]); - - let cursor = create_mock_cursor(trie_nodes); - let mut verifier = SingleVerifier::new(None, cursor).unwrap(); - let mut outputs = Vec::new(); - - // The depth-first iterator produces in post-order: 0x1, 0x2, 0x3, root - // We only provide 0x1 and 0x3, skipping 0x2 and root - verifier.next(&mut outputs, Nibbles::from_nibbles([0x1]), node1).unwrap(); - verifier.next(&mut outputs, Nibbles::from_nibbles([0x3]), node3).unwrap(); - verifier.finalize(&mut outputs).unwrap(); - - // Should have two "extra" outputs for nodes in the trie that we skipped - if outputs.len() != 2 { - eprintln!("Expected 2 outputs, got {}:", outputs.len()); - for inc in &outputs { - eprintln!(" {:?}", inc); - } - } - assert_eq!(outputs.len(), 2); - assert_matches!( - &outputs[0], - Output::AccountExtra(path, node) - if *path == Nibbles::from_nibbles([0x2]) && *node == node2 - ); - assert_matches!( - &outputs[1], - Output::AccountExtra(path, node) - if *path == Nibbles::new() && *node == node_root - ); - } - - #[test] - fn test_single_verifier_finalize() { - // Test finalize marks all remaining nodes as extra - let node_root = test_branch_node(0b1110, 0, 0b1110, vec![]); // root has children at 1, 2, 3 - let node1 = test_branch_node(0b0001, 0, 0b0001, vec![]); - let node2 = test_branch_node(0b0010, 0, 0b0010, vec![]); - let node3 = test_branch_node(0b0100, 0, 0b0100, vec![]); - - let trie_nodes = BTreeMap::from([ - (Nibbles::new(), node_root.clone()), - (Nibbles::from_nibbles([0x1]), node1.clone()), - (Nibbles::from_nibbles([0x2]), node2.clone()), - (Nibbles::from_nibbles([0x3]), node3.clone()), - ]); - - let cursor = create_mock_cursor(trie_nodes); - let mut verifier = SingleVerifier::new(None, cursor).unwrap(); - let mut outputs = Vec::new(); - - // The depth-first iterator produces in post-order: 0x1, 0x2, 0x3, root - // Process first two nodes correctly - verifier.next(&mut outputs, Nibbles::from_nibbles([0x1]), node1).unwrap(); - verifier.next(&mut outputs, Nibbles::from_nibbles([0x2]), node2).unwrap(); - assert!(outputs.is_empty()); - - // Finalize - should mark remaining nodes (0x3 and root) as extra - verifier.finalize(&mut outputs).unwrap(); - - // Should have two extra nodes - assert_eq!(outputs.len(), 2); - assert_matches!( - &outputs[0], - Output::AccountExtra(path, node) - if *path == Nibbles::from_nibbles([0x3]) && *node == node3 - ); - assert_matches!( - &outputs[1], - Output::AccountExtra(path, node) - if *path == Nibbles::new() && *node == node_root - ); - } - - #[test] - fn test_single_verifier_storage_trie() { - // Test SingleVerifier for storage trie (with account set) - let account = B256::repeat_byte(42u8); - let node = test_branch_node(0b1111, 0, 0b1111, vec![B256::repeat_byte(1u8)]); - - let trie_nodes = BTreeMap::from([(Nibbles::from_nibbles([0x1]), node)]); - - let cursor = create_mock_cursor(trie_nodes); - let mut verifier = SingleVerifier::new(Some(account), cursor).unwrap(); - let mut outputs = Vec::new(); - - // Call next with missing node - let missing_node = test_branch_node(0b0101, 0b0001, 0b0100, vec![B256::repeat_byte(2u8)]); - verifier.next(&mut outputs, Nibbles::from_nibbles([0x0]), missing_node.clone()).unwrap(); - - // Should produce StorageMissing, not AccountMissing - assert_eq!(outputs.len(), 1); - assert_matches!( - &outputs[0], - Output::StorageMissing(acc, path, node) - if *acc == account && *path == Nibbles::from_nibbles([0x0]) && *node == missing_node - ); - } - - #[test] - fn test_single_verifier_empty_trie() { - // Test with empty trie cursor - let trie_nodes = BTreeMap::new(); - let cursor = create_mock_cursor(trie_nodes); - let mut verifier = SingleVerifier::new(None, cursor).unwrap(); - let mut outputs = Vec::new(); - - // Any node should be marked as missing - let node = test_branch_node(0b1111, 0, 0b1111, vec![B256::repeat_byte(1u8)]); - verifier.next(&mut outputs, Nibbles::from_nibbles([0x1]), node.clone()).unwrap(); - - assert_eq!(outputs.len(), 1); - assert_matches!( - &outputs[0], - Output::AccountMissing(path, n) - if *path == Nibbles::from_nibbles([0x1]) && *n == node - ); - } - - #[test] - fn test_single_verifier_depth_first_ordering() { - // Test that nodes must be provided in depth-first order - // Create nodes with proper parent-child relationships - let node_root = test_branch_node(0b0110, 0, 0b0110, vec![]); // root has children at 1 and 2 - let node1 = test_branch_node(0b0110, 0, 0b0110, vec![]); // 0x1 has children at 1 and 2 - let node11 = test_branch_node(0b0001, 0, 0b0001, vec![]); // 0x11 is a leaf - let node12 = test_branch_node(0b0010, 0, 0b0010, vec![]); // 0x12 is a leaf - let node2 = test_branch_node(0b0100, 0, 0b0100, vec![]); // 0x2 is a leaf - - // The depth-first iterator will iterate from the root in this order: - // root -> 0x1 -> 0x11, 0x12 (children of 0x1), then 0x2 - // But because of depth-first, we get: root, 0x1, 0x11, 0x12, 0x2 - let trie_nodes = BTreeMap::from([ - (Nibbles::new(), node_root.clone()), // root - (Nibbles::from_nibbles([0x1]), node1.clone()), // 0x1 - (Nibbles::from_nibbles([0x1, 0x1]), node11.clone()), // 0x11 - (Nibbles::from_nibbles([0x1, 0x2]), node12.clone()), // 0x12 - (Nibbles::from_nibbles([0x2]), node2.clone()), // 0x2 - ]); - - let cursor = create_mock_cursor(trie_nodes); - let mut verifier = SingleVerifier::new(None, cursor).unwrap(); - let mut outputs = Vec::new(); - - // The depth-first iterator produces nodes in post-order (children before parents) - // Order: 0x11, 0x12, 0x1, 0x2, root - verifier.next(&mut outputs, Nibbles::from_nibbles([0x1, 0x1]), node11).unwrap(); - verifier.next(&mut outputs, Nibbles::from_nibbles([0x1, 0x2]), node12).unwrap(); - verifier.next(&mut outputs, Nibbles::from_nibbles([0x1]), node1).unwrap(); - verifier.next(&mut outputs, Nibbles::from_nibbles([0x2]), node2).unwrap(); - verifier.next(&mut outputs, Nibbles::new(), node_root).unwrap(); - verifier.finalize(&mut outputs).unwrap(); - - // All should match, no outputs - if !outputs.is_empty() { - eprintln!( - "Test test_single_verifier_depth_first_ordering failed with {} outputs:", - outputs.len() - ); - for inc in &outputs { - eprintln!(" {:?}", inc); - } - } - assert!(outputs.is_empty()); - } - - #[test] - fn test_single_verifier_wrong_depth_first_order() { - // Test that providing nodes in wrong order produces outputs - // Create a trie with parent-child relationship - let node_root = test_branch_node(0b0010, 0, 0b0010, vec![]); // root has child at 1 - let node1 = test_branch_node(0b0010, 0, 0b0010, vec![]); // 0x1 has child at 1 - let node11 = test_branch_node(0b0001, 0, 0b0001, vec![]); // 0x11 is a leaf - - let trie_nodes = BTreeMap::from([ - (Nibbles::new(), node_root.clone()), - (Nibbles::from_nibbles([0x1]), node1.clone()), - (Nibbles::from_nibbles([0x1, 0x1]), node11.clone()), - ]); - - let cursor = create_mock_cursor(trie_nodes); - let mut verifier = SingleVerifier::new(None, cursor).unwrap(); - let mut outputs = Vec::new(); - - // Process in WRONG order (skip root, provide child before processing all nodes correctly) - // The iterator will produce: root, 0x1, 0x11 - // But we provide: 0x11, root, 0x1 (completely wrong order) - verifier.next(&mut outputs, Nibbles::from_nibbles([0x1, 0x1]), node11).unwrap(); - verifier.next(&mut outputs, Nibbles::new(), node_root).unwrap(); - verifier.next(&mut outputs, Nibbles::from_nibbles([0x1]), node1).unwrap(); - - // Should have outputs since we provided them in wrong order - assert!(!outputs.is_empty()); - } - - #[test] - fn test_single_verifier_complex_depth_first() { - // Test a complex tree structure with depth-first ordering - // Build a tree structure with proper parent-child relationships - let node_root = test_branch_node(0b0110, 0, 0b0110, vec![]); // root: children at nibbles 1 - // and 2 - let node1 = test_branch_node(0b0110, 0, 0b0110, vec![]); // 0x1: children at nibbles 1 and 2 - let node11 = test_branch_node(0b0110, 0, 0b0110, vec![]); // 0x11: children at nibbles 1 and - // 2 - let node111 = test_branch_node(0b0001, 0, 0b0001, vec![]); // 0x111: leaf - let node112 = test_branch_node(0b0010, 0, 0b0010, vec![]); // 0x112: leaf - let node12 = test_branch_node(0b0100, 0, 0b0100, vec![]); // 0x12: leaf - let node2 = test_branch_node(0b0010, 0, 0b0010, vec![]); // 0x2: child at nibble 1 - let node21 = test_branch_node(0b0001, 0, 0b0001, vec![]); // 0x21: leaf - - // Create the trie structure - let trie_nodes = BTreeMap::from([ - (Nibbles::new(), node_root.clone()), - (Nibbles::from_nibbles([0x1]), node1.clone()), - (Nibbles::from_nibbles([0x1, 0x1]), node11.clone()), - (Nibbles::from_nibbles([0x1, 0x1, 0x1]), node111.clone()), - (Nibbles::from_nibbles([0x1, 0x1, 0x2]), node112.clone()), - (Nibbles::from_nibbles([0x1, 0x2]), node12.clone()), - (Nibbles::from_nibbles([0x2]), node2.clone()), - (Nibbles::from_nibbles([0x2, 0x1]), node21.clone()), - ]); - - let cursor = create_mock_cursor(trie_nodes); - let mut verifier = SingleVerifier::new(None, cursor).unwrap(); - let mut outputs = Vec::new(); - - // The depth-first iterator produces nodes in post-order (children before parents) - // Order: 0x111, 0x112, 0x11, 0x12, 0x1, 0x21, 0x2, root - verifier.next(&mut outputs, Nibbles::from_nibbles([0x1, 0x1, 0x1]), node111).unwrap(); - verifier.next(&mut outputs, Nibbles::from_nibbles([0x1, 0x1, 0x2]), node112).unwrap(); - verifier.next(&mut outputs, Nibbles::from_nibbles([0x1, 0x1]), node11).unwrap(); - verifier.next(&mut outputs, Nibbles::from_nibbles([0x1, 0x2]), node12).unwrap(); - verifier.next(&mut outputs, Nibbles::from_nibbles([0x1]), node1).unwrap(); - verifier.next(&mut outputs, Nibbles::from_nibbles([0x2, 0x1]), node21).unwrap(); - verifier.next(&mut outputs, Nibbles::from_nibbles([0x2]), node2).unwrap(); - verifier.next(&mut outputs, Nibbles::new(), node_root).unwrap(); - verifier.finalize(&mut outputs).unwrap(); - - // All should match, no outputs - if !outputs.is_empty() { - eprintln!( - "Test test_single_verifier_complex_depth_first failed with {} outputs:", - outputs.len() - ); - for inc in &outputs { - eprintln!(" {:?}", inc); - } - } - assert!(outputs.is_empty()); - } -} diff --git a/crates/trie/trie/src/verify/compare.rs b/crates/trie/trie/src/verify/compare.rs new file mode 100644 index 00000000000..43e774ca25f --- /dev/null +++ b/crates/trie/trie/src/verify/compare.rs @@ -0,0 +1,141 @@ +//! Depth-first comparison of persisted trie nodes against recomputed nodes. + +use super::Output; +use crate::{ + trie_cursor::{ + depth_first::{self, DepthFirstTrieIterator}, + TrieCursor, + }, + Nibbles, +}; +use alloy_primitives::B256; +use alloy_trie::BranchNodeCompact; +use reth_storage_errors::db::DatabaseError; +use std::cmp::Ordering; +use tracing::trace; + +/// Verifies the contents of a trie table against some other data source which is able to produce +/// stored trie nodes. +#[derive(Debug)] +pub(super) struct SingleVerifier { + account: Option, // None for accounts trie + trie_iter: I, + curr: Option<(Nibbles, BranchNodeCompact)>, +} + +impl SingleVerifier> { + pub(super) fn new(account: Option, trie_cursor: C) -> Result { + let mut trie_iter = DepthFirstTrieIterator::new(trie_cursor); + let curr = trie_iter.next().transpose()?; + Ok(Self { account, trie_iter, curr }) + } +} + +impl SingleVerifier +where + I: Iterator>, +{ + const fn output_extra(&self, path: Nibbles, node: BranchNodeCompact) -> Output { + if let Some(account) = self.account { + Output::StorageExtra(account, path, node) + } else { + Output::AccountExtra(path, node) + } + } + + const fn output_wrong( + &self, + path: Nibbles, + expected: BranchNodeCompact, + found: BranchNodeCompact, + ) -> Output { + if let Some(account) = self.account { + Output::StorageWrong { account, path, expected, found } + } else { + Output::AccountWrong { path, expected, found } + } + } + + const fn output_missing(&self, path: Nibbles, node: BranchNodeCompact) -> Output { + if let Some(account) = self.account { + Output::StorageMissing(account, path, node) + } else { + Output::AccountMissing(path, node) + } + } + + /// Called with the next path and node in the canonical sequence of stored trie nodes. Will + /// emit findings immediately if walking the trie cursor produces inconsistent data. + /// Returning `false` from the callback cancels comparison before advancing the cursor again. + /// + /// `next` must be called with paths in depth-first order. Returns `true` on completion and + /// `false` on cancellation. The verifier must not be reused after cancellation. + pub(super) fn next( + &mut self, + emit: &mut impl FnMut(Output) -> bool, + path: Nibbles, + node: BranchNodeCompact, + ) -> Result { + loop { + // `curr` is None only if the end of the iterator has been reached. Any further nodes + // found must be considered missing. + if self.curr.is_none() { + return Ok(emit(self.output_missing(path, node))) + } + + let (curr_path, curr_node) = self.curr.as_ref().expect("not None"); + trace!(target: "trie::verify", account=?self.account, ?curr_path, ?path, "Current cursor node"); + + // Use depth-first ordering for comparison + match depth_first::cmp(&path, curr_path) { + Ordering::Less => { + // If the given path comes before the cursor's current path in depth-first + // order, then the given path was not produced by the cursor. + return Ok(emit(self.output_missing(path, node))) + } + Ordering::Equal => { + // If the current path matches the given one (happy path) but the nodes + // aren't equal then we produce a wrong node. Either way we want to move the + // iterator forward. + if *curr_node != node && !emit(self.output_wrong(path, node, curr_node.clone())) + { + return Ok(false) + } + self.curr = self.trie_iter.next().transpose()?; + return Ok(true); + } + Ordering::Greater => { + // If the given path comes after the current path in depth-first order, + // it means the cursor's path was not found by the caller (otherwise it would + // have hit the equal case) and so is extraneous. + if !emit(self.output_extra(*curr_path, curr_node.clone())) { + return Ok(false); + } + self.curr = self.trie_iter.next().transpose()?; + // back to the top of the loop to check the latest `self.curr` value against the + // given path/node. + } + } + } + } + + /// Must be called once there are no more calls to `next` to be made. All further nodes produced + /// by the iterator will be considered extraneous and emitted individually. Returning `false` + /// from the callback stops comparison immediately; the verifier must not be reused after + /// cancellation. Returns `true` on completion and `false` on cancellation. + pub(super) fn finalize( + &mut self, + emit: &mut impl FnMut(Output) -> bool, + ) -> Result { + loop { + if let Some((curr_path, curr_node)) = self.curr.take() { + if !emit(self.output_extra(curr_path, curr_node)) { + return Ok(false); + } + self.curr = self.trie_iter.next().transpose()?; + } else { + return Ok(true); + } + } + } +} diff --git a/crates/trie/trie/src/verify/mod.rs b/crates/trie/trie/src/verify/mod.rs new file mode 100644 index 00000000000..ea6af402125 --- /dev/null +++ b/crates/trie/trie/src/verify/mod.rs @@ -0,0 +1,303 @@ +//! Parallel verification of persisted account and storage tries. + +use self::{ + compare::SingleVerifier, + pipeline::{StoragePipeline, WorkerContext}, +}; +use crate::{ + hashed_cursor::{HashedCursor, HashedStorageCursor}, + trie_cursor::{ + depth_first::{self, DepthFirstTrieIterator}, + TrieCursor, TrieStorageCursor, + }, + Nibbles, +}; +use alloy_primitives::{B256, U256}; +use alloy_rlp::Encodable; +use alloy_trie::BranchNodeCompact; +use crossbeam_channel::{Receiver, Sender}; +use reth_execution_errors::StateRootError; +use reth_primitives_traits::Account; +use reth_storage_errors::db::DatabaseError; +use std::{ + sync::Arc, + thread::{self, JoinHandle}, +}; + +mod compare; +mod pipeline; +mod storage; +pub use storage::{StorageJob, StorageWorker}; + +/// Compares the persisted account and storage trie tables against tries recomputed from the hashed +/// state tables, yielding an [`Output`] for every inconsistency found. +/// +/// The hashed tables are the source of truth. Storage tries are recomputed in parallel, so storage +/// findings are not ordered relative to account findings or to [`Output::Progress`]. Iteration +/// stops after the first error. Only hashed account rows contribute account leaves; orphan storage +/// tries are still verified independently. +/// +/// Storage work ahead of the account walk is bounded by the configured storage lookahead. +/// Findings use a bounded channel shared by account and storage producers, so a slow consumer +/// applies backpressure without buffering an entire storage trie while waiting for its root. +#[derive(Debug)] +pub struct Verifier { + events: Receiver>, + // The sole sender is dropped to broadcast cancellation to every producer. + cancel: Option>, + // Retain account cursors until all producers have joined, before backing transactions drop. + background: Option>>, +} + +const FINDINGS_CAPACITY: usize = 1024; + +const ACCOUNT_UPDATE_THRESHOLD: usize = 1024; + +/// Default maximum number of storage roots computed or in flight ahead of the account walk. +pub const DEFAULT_STORAGE_LOOKAHEAD: usize = 1_000_000; + +impl Verifier +where + T: TrieCursor + Send + 'static, + H: HashedCursor + Send + 'static, +{ + /// Starts verification and its background storage root computation. + /// + /// - `trie` reads the account trie table. + /// - `accounts` reads the hashed accounts table in ascending address order. + /// - `storage_addresses` yields a job for each address present in hashed storage or stored + /// storage tries exactly once, in ascending address order, including addresses without an + /// account row. Jobs with hashed storage must request a root even without an account row. It + /// is walked on a background thread to schedule storage work ahead of `accounts`. + /// - `storage_lookahead` bounds storage jobs whose requested roots have not been consumed, + /// including storage without a hashed account row. + /// - `storage_workers` sets the storage parallelism, one thread per worker. + /// + /// Storage cursors may start at any hashed address. All cursors must read the same database + /// version, and any transactions backing them must stay open until the verifier is dropped. + /// The account walk runs on a producer thread. Account and storage findings share a bounded + /// channel, consumed by this iterator independently of storage root waits. Account cursors + /// must be transferable to that thread. Dropping the verifier cancels and joins all work. + /// + /// # Errors + /// + /// Returns an error if the account trie cannot be read or the account thread cannot be + /// spawned. Errors starting storage threads are yielded by the iterator. + /// + /// # Panics + /// + /// Panics if `storage_workers` is empty or `storage_lookahead` is zero. + pub fn new( + trie: T, + accounts: H, + storage_addresses: L, + storage_workers: Vec>, + storage_lookahead: usize, + ) -> Result + where + L: Iterator> + Send + 'static, + W: TrieStorageCursor + Send + 'static, + C: HashedStorageCursor + Send + 'static, + TX: Send + 'static, + { + let account_verifier = SingleVerifier::new(None, trie)?; + assert!(!storage_workers.is_empty(), "verification requires storage workers"); + assert!(storage_lookahead > 0, "storage lookahead must be positive"); + let (events_tx, events) = crossbeam_channel::bounded(FINDINGS_CAPACITY); + let (cancel, cancel_rx) = crossbeam_channel::bounded(0); + let context = WorkerContext::new(events_tx, cancel_rx); + let mut account = AccountVerifier { + accounts, + account_verifier, + hash_builder: crate::HashBuilder::default().with_updates(true), + account_rlp: Vec::with_capacity(crate::TRIE_ACCOUNT_RLP_MAX_SIZE), + }; + let span = tracing::Span::current(); + let background = thread::Builder::new() + .name("trie-accounts".into()) + .spawn(move || { + let _entered = span.enter(); + match StoragePipeline::spawn( + storage_addresses, + storage_workers, + context.clone(), + storage_lookahead, + ) { + Ok(mut pipeline) => { + // Report failures before joining: the consumer cancels all producers on + // error. + context.run(|| account.run(&mut pipeline, &context)); + pipeline.finish(); + } + Err(error) => context.run(|| Err(error.into())), + } + account + }) + .map_err(|error| DatabaseError::Custom(Arc::new(error)))?; + Ok(Self { events, cancel: Some(cancel), background: Some(background) }) + } +} + +impl Verifier { + fn stop(&mut self) { + self.cancel.take(); + if let Some(background) = self.background.take() && + let Err(panic) = background.join() && + !thread::panicking() + { + std::panic::resume_unwind(panic); + } + } +} + +impl Iterator for Verifier { + type Item = Result; + + fn next(&mut self) -> Option { + self.cancel.as_ref()?; + match self.events.recv() { + Ok(Ok(output)) => Some(Ok(output)), + Ok(Err(error)) => { + self.stop(); + Some(Err(error)) + } + Err(_) => { + self.stop(); + None + } + } + } +} + +impl Drop for Verifier { + fn drop(&mut self) { + self.stop(); + } +} + +/// Account producer state, returned from its thread so cursor lifetimes remain tied to `Verifier`. +#[derive(Debug)] +struct AccountVerifier { + accounts: H, + account_verifier: SingleVerifier>, + hash_builder: crate::HashBuilder, + account_rlp: Vec, +} + +impl AccountVerifier +where + T: TrieCursor, + H: HashedCursor, +{ + fn run( + &mut self, + pipeline: &mut StoragePipeline, + context: &WorkerContext, + ) -> Result<(), StateRootError> { + let mut next_account = self.accounts.seek(B256::ZERO)?; + loop { + if context.is_cancelled() { + return Ok(()); + } + // The scheduler announces storage addresses in order before dispatching their jobs. + let next_root = pipeline.next_root(next_account.as_ref().map(|(address, _)| *address)); + if context.is_cancelled() { + return Ok(()); + } + let (address, root) = match (next_account.as_ref(), next_root) { + (_, Some(storage)) => storage, + (Some((address, _)), None) => (*address, alloy_consensus::EMPTY_ROOT_HASH), + (None, None) => break, + }; + let account = match next_account.take() { + Some((key, account)) if key == address => { + next_account = self.accounts.next()?; + account + } + next => { + next_account = next; + // Canonical state roots only include leaves present in hashed accounts. + // Orphan storage is still verified, but cannot create an account leaf. + continue; + } + }; + self.account_rlp.clear(); + account.into_trie_account(root).encode(&mut self.account_rlp); + self.hash_builder.add_leaf(Nibbles::unpack(address), &self.account_rlp); + if self.hash_builder.updates_len() >= ACCOUNT_UPDATE_THRESHOLD && + !self.verify_account_updates(context)? + { + return Ok(()); + } + } + // Computing the root flushes the remaining branch node updates. + self.hash_builder.root(); + if !self.verify_account_updates(context)? { + return Ok(()); + } + self.account_verifier.finalize(&mut |output| context.report_one(output))?; + Ok(()) + } + + fn verify_account_updates(&mut self, context: &WorkerContext) -> Result { + let (builder, updates) = std::mem::take(&mut self.hash_builder).split(); + self.hash_builder = builder.with_updates(true); + // The account trie root is calculated but never persisted. Updates are sorted depth-first + // because that is the order in which the `HashBuilder` completes branch nodes, so each + // batch continues where the previous one ended. + let mut nodes = + updates.into_iter().filter(|(path, _)| !path.is_empty()).collect::>(); + nodes.sort_unstable_by(|a, b| depth_first::cmp(&a.0, &b.0)); + for (path, node) in nodes { + if !self.account_verifier.next(&mut |output| context.report_one(output), path, node)? || + !context.report_one(Output::Progress(path)) + { + return Ok(false); + } + } + Ok(true) + } +} + +/// Output describes an inconsistency found when comparing the hashed state tables +/// ([`HashedCursor`]) with that of the trie tables ([`TrieCursor`]). The hashed +/// tables are considered the source of truth; outputs are on the part of the trie tables. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Output { + /// An extra account node was found. + AccountExtra(Nibbles, BranchNodeCompact), + /// An extra storage node was found. + StorageExtra(B256, Nibbles, BranchNodeCompact), + /// An account node had the wrong value. + AccountWrong { + /// Path of the node + path: Nibbles, + /// The node's expected value. + expected: BranchNodeCompact, + /// The node's found value. + found: BranchNodeCompact, + }, + /// A storage node had the wrong value. + StorageWrong { + /// The account the storage trie belongs to. + account: B256, + /// Path of the node + path: Nibbles, + /// The node's expected value. + expected: BranchNodeCompact, + /// The node's found value. + found: BranchNodeCompact, + }, + /// An account node was missing. + AccountMissing(Nibbles, BranchNodeCompact), + /// A storage node was missing. + StorageMissing(B256, Nibbles, BranchNodeCompact), + /// Progress indicator with the path of the last verified account trie node. + Progress(Nibbles), +} + +/// Returns the default number of [`StorageWorker`]s to pass to [`Verifier::new`]: one per available +/// CPU, or one if the CPU count cannot be determined. +pub fn default_storage_workers() -> usize { + thread::available_parallelism().map_or(1, usize::from) +} diff --git a/crates/trie/trie/src/verify/pipeline.rs b/crates/trie/trie/src/verify/pipeline.rs new file mode 100644 index 00000000000..2be49278fb4 --- /dev/null +++ b/crates/trie/trie/src/verify/pipeline.rs @@ -0,0 +1,257 @@ +//! Ordered storage scheduling, root delivery, shared findings, and background task lifetime. + +use super::{Output, StorageJob, StorageWorker}; +use crate::{hashed_cursor::HashedStorageCursor, trie_cursor::TrieStorageCursor}; +use alloy_primitives::{B256, U256}; +use crossbeam_channel::{Receiver, Sender}; +use reth_execution_errors::StateRootError; +use reth_storage_errors::db::DatabaseError; +use std::{ + panic::{catch_unwind, AssertUnwindSafe}, + sync::Arc, + thread::{self, JoinHandle}, +}; + +/// Owns storage scheduling and one pending root announcement for the account producer. +#[derive(Debug)] +pub(super) struct StoragePipeline { + context: WorkerContext, + background: Option>, + announcements: Receiver<(B256, Receiver)>, + next: Option<(B256, Receiver)>, +} + +impl StoragePipeline { + /// Spawns the background thread which walks `addresses` and verifies their storage on one + /// thread per worker. + /// + /// # Panics + /// + /// Panics if `workers` is empty or `storage_lookahead` is zero. + pub(super) fn spawn( + addresses: L, + workers: Vec>, + context: WorkerContext, + storage_lookahead: usize, + ) -> Result + where + L: Iterator> + Send + 'static, + T: TrieStorageCursor + Send + 'static, + H: HashedStorageCursor + Send + 'static, + TX: Send + 'static, + { + assert!(!workers.is_empty(), "verification requires storage workers"); + assert!(storage_lookahead > 0, "storage lookahead must be positive"); + // The account producer retains one announcement outside the queue. Reserving its + // slot here bounds all dispatched roots, including that lookahead, by the limit. + let (announcements_tx, announcements) = crossbeam_channel::bounded(storage_lookahead - 1); + let scheduler = StorageScheduler { addresses, workers, context: context.clone() }; + let span = tracing::Span::current(); + let background = thread::Builder::new() + .name("trie-lookahead".into()) + .spawn(move || { + let _entered = span.enter(); + scheduler.run(announcements_tx); + }) + .map_err(|error| DatabaseError::Custom(Arc::new(error)))?; + Ok(Self { context, background: Some(background), announcements, next: None }) + } + + /// Returns the next root in address order up to and including `up_to`, or all remaining + /// roots if `up_to` is `None`. Returns `None` when the bound is reached, scanning completes + /// without another root, or verification is cancelled. Findings are consumed independently. + /// A missing worker result waits for cancellation by the findings consumer after it receives + /// the worker error, rather than replacing the original error with a channel error. + pub(super) fn next_root(&mut self, up_to: Option) -> Option<(B256, B256)> { + if self.context.is_cancelled() { + return None; + } + if self.next.is_none() { + self.next = crossbeam_channel::select! { + recv(self.announcements) -> announcement => announcement.ok(), + recv(self.context.cancel) -> _ => return None, + }; + } + let Some((address, root)) = &self.next else { return None }; + if up_to.is_some_and(|bound| *address > bound) { + return None; + } + let root = crossbeam_channel::select! { + recv(root) -> root => root, + recv(self.context.cancel) -> _ => return None, + }; + match root { + Ok(root) => { + let address = *address; + self.next = None; + Some((address, root)) + } + Err(_) => { + // A failing worker drops its root sender before WorkerContext::run reports the + // original error. The findings consumer cancels us after receiving that error; + // do not race it with a synthetic channel error or skip the missing root. + match self.context.cancel.recv() { + Ok(()) | Err(_) => None, + } + } + } + } + + /// Joins storage producers while the caller's iterator continues consuming findings. + pub(super) fn finish(&mut self) { + self.join(); + } + + fn join(&mut self) { + // Re-raising while already unwinding would abort the process. + if let Some(background) = self.background.take() && + let Err(panic) = background.join() && + !thread::panicking() + { + std::panic::resume_unwind(panic); + } + } +} + +impl Drop for StoragePipeline { + fn drop(&mut self) { + self.join(); + } +} + +/// Body of the background thread: walks storage addresses and keeps the workers supplied with +/// jobs. +struct StorageScheduler { + addresses: L, + workers: Vec>, + context: WorkerContext, +} + +impl StorageScheduler +where + L: Iterator>, + T: TrieStorageCursor + Send, + H: HashedStorageCursor + Send, + TX: Send, +{ + fn run(mut self, announcements: Sender<(B256, Receiver)>) { + let context = self.context.clone(); + // Retain the sender across setup failures until their error has been reported. + let mut announcements = Some(announcements); + context.run(|| { + let worker_count = self.workers.len(); + let pool = reth_tasks::WorkerPool::new(worker_count, "trie-verify"); + // Reserve backlog space before dispatch so a slow early root can always finish + // without competing with later results for space. + let (jobs_tx, jobs_rx) = crossbeam_channel::bounded(worker_count * 2); + pool.in_place_scope(|scope| { + for worker in self.workers.drain(..) { + let jobs = jobs_rx.clone(); + let context = self.context.clone(); + let span = tracing::Span::current(); + scope.spawn(move |_| { + let _entered = span.enter(); + context.run(|| worker.run(jobs, &context)); + }); + } + drop(jobs_rx); + // Close announcements before joining workers: account EOF must not depend on + // workers finishing their findings sends. + // Keep scan failure ordered before EOF, including when `produce` panics. + if let Some(sender) = &announcements { + context.run(|| self.produce(jobs_tx, sender)); + } + drop(announcements.take()); + }); + Ok(()) + }); + } + + fn produce( + &mut self, + jobs: Sender<(B256, Option>)>, + announcements: &Sender<(B256, Receiver)>, + ) -> Result<(), StateRootError> { + for job in self.addresses.by_ref() { + let job = job?; + let root = if job.root_required { + let (sender, receiver) = crossbeam_channel::bounded(1); + // Announce in scan order before dispatch, regardless of completion order. + if !self.context.send(announcements, (job.address, receiver)) { + return Ok(()); + } + Some(sender) + } else { + None + }; + if !self.context.send(&jobs, (job.address, root)) { + return Ok(()); + } + } + Ok(()) + } +} + +/// Shared cancellation and reporting policy for background threads. +/// +/// Verification is cancelled once the `cancel` sender is dropped; nothing is ever sent on it. +#[derive(Clone, Debug)] +pub(super) struct WorkerContext { + events: Sender>, + cancel: Receiver<()>, +} + +impl WorkerContext { + pub(super) const fn new( + events: Sender>, + cancel: Receiver<()>, + ) -> Self { + Self { events, cancel } + } + + /// Sends `value`, returning `false` if it was not sent because verification was cancelled or + /// `sender` disconnected. + pub(super) fn send(&self, sender: &Sender, value: T) -> bool { + crossbeam_channel::select! { + send(sender, value) -> result => result.is_ok(), + recv(self.cancel) -> _ => false, + } + } + + /// Receives the next job. Returns `None` once `jobs` is exhausted, and may return `None` early + /// if verification was cancelled. + pub(super) fn receive(&self, jobs: &Receiver) -> Option { + crossbeam_channel::select! { + recv(jobs) -> job => job.ok(), + recv(self.cancel) -> _ => None, + } + } + + /// Returns whether verification was cancelled. + pub(super) fn is_cancelled(&self) -> bool { + self.cancel.try_recv() == Err(crossbeam_channel::TryRecvError::Disconnected) + } + + /// Reports one finding, applying backpressure until it is consumed or verification stops. + pub(super) fn report_one(&self, output: Output) -> bool { + self.send(&self.events, Ok(output)) + } + + /// Runs `work`, reporting its error or panic as a verification error. + pub(super) fn run(&self, work: impl FnOnce() -> Result<(), StateRootError>) { + let result = catch_unwind(AssertUnwindSafe(work)).unwrap_or_else(|panic| { + let message = panic + .downcast_ref::<&str>() + .copied() + .or_else(|| panic.downcast_ref::().map(String::as_str)) + .unwrap_or("unknown panic"); + Err(DatabaseError::Other(format!("trie verification thread panicked: {message}")) + .into()) + }); + // The error is only lost if verification was already cancelled, in which case the + // verifier has stopped reading results. + if let Err(error) = result { + self.send(&self.events, Err(error)); + } + } +} diff --git a/crates/trie/trie/src/verify/storage.rs b/crates/trie/trie/src/verify/storage.rs new file mode 100644 index 00000000000..77ec0ded634 --- /dev/null +++ b/crates/trie/trie/src/verify/storage.rs @@ -0,0 +1,146 @@ +//! Storage verification using one worker's long-lived transaction and reusable cursors. + +use super::{compare::SingleVerifier, pipeline::WorkerContext}; +use crate::{ + hashed_cursor::HashedStorageCursor, + trie::{StorageRootCalculation, DEFAULT_INTERMEDIATE_THRESHOLD}, + trie_cursor::{depth_first, noop::NoopStorageTrieCursor, TrieStorageCursor}, + StorageRootProgress, +}; +use alloy_consensus::EMPTY_ROOT_HASH; +use alloy_primitives::{B256, U256}; +use crossbeam_channel::{Receiver, Sender}; +use reth_execution_errors::StateRootError; + +#[cfg(feature = "metrics")] +use crate::{metrics::TrieRootMetrics, trie::TrieType}; + +/// Cursors for one parallel storage verification thread of a [`Verifier`](super::Verifier). +/// +/// Both cursors may start at any hashed address. `tx` is the transaction backing both cursors; it +/// is held without being read and dropped after them. +#[derive(Debug)] +pub struct StorageWorker { + /// Cursor over the storage trie table. + pub trie: T, + /// Cursor over the hashed storage table. + pub hashed: H, + /// Transaction backing `trie` and `hashed`. + pub tx: TX, +} + +impl StorageWorker +where + T: TrieStorageCursor, + H: HashedStorageCursor, +{ + /// Verifies the storage of each address received from `jobs`, completing requested roots + /// and reporting findings through `context`. Returns once `jobs` is exhausted or + /// verification is cancelled. + pub(super) fn run( + mut self, + jobs: Receiver<(B256, Option>)>, + context: &WorkerContext, + ) -> Result<(), StateRootError> { + #[cfg(feature = "metrics")] + let metrics = TrieRootMetrics::new(TrieType::Storage); + + loop { + let Some((address, root_sender)) = context.receive(&jobs) else { return Ok(()) }; + self.hashed.set_hashed_address(address); + self.trie.set_hashed_address(address); + let root = if self.hashed.is_storage_empty()? { + let mut verifier = SingleVerifier::new(Some(address), &mut self.trie)?; + if !verifier.finalize(&mut |output| context.report_one(output))? { + return Ok(()); + } + EMPTY_ROOT_HASH + } else { + let Some(root) = self.verify_storage( + address, + context, + #[cfg(feature = "metrics")] + &metrics, + )? + else { + return Ok(()) + }; + root + }; + if let Some(sender) = root_sender && + !context.send(&sender, root) + { + return Ok(()); + } + } + } + + /// Recomputes the storage trie of `address` from hashed state alone and compares it against + /// the storage trie table. Work is done in bounded rounds so memory stays bounded for large + /// storages and cancellation is observed between rounds. + /// + /// Returns `None` if verification was cancelled. + fn verify_storage( + &mut self, + address: B256, + context: &WorkerContext, + #[cfg(feature = "metrics")] metrics: &TrieRootMetrics, + ) -> Result, StateRootError> { + let mut verifier = SingleVerifier::new(Some(address), &mut self.trie)?; + let mut state = None; + loop { + if context.is_cancelled() { + return Ok(None); + } + let result = StorageRootCalculation { + hashed_address: address, + prefix_set: Default::default(), + previous_state: state.take(), + walk_all_changed_branch_children: false, + threshold: DEFAULT_INTERMEDIATE_THRESHOLD, + retain_updates: true, + } + .calculate( + &mut NoopStorageTrieCursor::default(), + &mut self.hashed, + #[cfg(feature = "metrics")] + metrics, + )?; + let (root, updates) = match result { + StorageRootProgress::Complete(root, _, updates) => (Some(root), updates), + StorageRootProgress::Progress(next, _, updates) => { + state = Some(*next); + (None, updates) + } + }; + let mut nodes = updates.storage_nodes.into_iter().collect::>(); + nodes.sort_unstable_by(|a, b| depth_first::cmp(&a.0, &b.0)); + let mut emit = |output| context.report_one(output); + for (path, node) in nodes { + if context.is_cancelled() { + return Ok(None); + } + if !verifier.next(&mut emit, path, node)? { + return Ok(None); + } + } + if let Some(root) = root { + if !verifier.finalize(&mut emit)? { + return Ok(None); + } + return Ok(Some(root)); + } + } + } +} + +/// Storage to verify, including whether the account walk needs its root. +#[derive(Debug)] +pub struct StorageJob { + /// Hashed address of the storage to verify. + pub address: B256, + /// Whether to return the computed root to the account walk. False still verifies the storage + /// trie, but discards its root and does not reserve space in the root backlog. Hashed storage + /// needs a root even without a hashed account row; only stale trie-only storage may omit it. + pub root_required: bool, +} diff --git a/docs/vocs/docs/pages/cli/reth.mdx b/docs/vocs/docs/pages/cli/reth.mdx index 11d01e15fea..f8f52300219 100644 --- a/docs/vocs/docs/pages/cli/reth.mdx +++ b/docs/vocs/docs/pages/cli/reth.mdx @@ -135,10 +135,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/config.mdx b/docs/vocs/docs/pages/cli/reth/config.mdx index 3bc41f2a8c4..ac795ff8dbb 100644 --- a/docs/vocs/docs/pages/cli/reth/config.mdx +++ b/docs/vocs/docs/pages/cli/reth/config.mdx @@ -120,10 +120,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db.mdx b/docs/vocs/docs/pages/cli/reth/db.mdx index 6e3515d9138..503987b76ec 100644 --- a/docs/vocs/docs/pages/cli/reth/db.mdx +++ b/docs/vocs/docs/pages/cli/reth/db.mdx @@ -256,10 +256,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/account-storage.mdx b/docs/vocs/docs/pages/cli/reth/db/account-storage.mdx index bd35d0e21e3..2cf26dddc67 100644 --- a/docs/vocs/docs/pages/cli/reth/db/account-storage.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/account-storage.mdx @@ -128,10 +128,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/checksum.mdx b/docs/vocs/docs/pages/cli/reth/db/checksum.mdx index 24e4f3d7c3b..fb246dfd574 100644 --- a/docs/vocs/docs/pages/cli/reth/db/checksum.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/checksum.mdx @@ -130,10 +130,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/checksum/mdbx.mdx b/docs/vocs/docs/pages/cli/reth/db/checksum/mdbx.mdx index d99646eea81..ce25a4460c1 100644 --- a/docs/vocs/docs/pages/cli/reth/db/checksum/mdbx.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/checksum/mdbx.mdx @@ -137,10 +137,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/checksum/rocksdb.mdx b/docs/vocs/docs/pages/cli/reth/db/checksum/rocksdb.mdx index 9eff0527589..54f3e7ae72c 100644 --- a/docs/vocs/docs/pages/cli/reth/db/checksum/rocksdb.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/checksum/rocksdb.mdx @@ -136,10 +136,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/checksum/static-file.mdx b/docs/vocs/docs/pages/cli/reth/db/checksum/static-file.mdx index d7a107c2ede..7082f94bddf 100644 --- a/docs/vocs/docs/pages/cli/reth/db/checksum/static-file.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/checksum/static-file.mdx @@ -145,10 +145,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/clear.mdx b/docs/vocs/docs/pages/cli/reth/db/clear.mdx index ab269646eb0..d77fca27e21 100644 --- a/docs/vocs/docs/pages/cli/reth/db/clear.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/clear.mdx @@ -129,10 +129,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/clear/mdbx.mdx b/docs/vocs/docs/pages/cli/reth/db/clear/mdbx.mdx index cb52a3fcd2b..048348f8f60 100644 --- a/docs/vocs/docs/pages/cli/reth/db/clear/mdbx.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/clear/mdbx.mdx @@ -128,10 +128,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/clear/static-file.mdx b/docs/vocs/docs/pages/cli/reth/db/clear/static-file.mdx index aba508cf211..6d75970afc5 100644 --- a/docs/vocs/docs/pages/cli/reth/db/clear/static-file.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/clear/static-file.mdx @@ -134,10 +134,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/copy.mdx b/docs/vocs/docs/pages/cli/reth/db/copy.mdx index 7a35a7b4cb3..effb5fcd325 100644 --- a/docs/vocs/docs/pages/cli/reth/db/copy.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/copy.mdx @@ -137,10 +137,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/diff.mdx b/docs/vocs/docs/pages/cli/reth/db/diff.mdx index 4086c9662fd..ef9410f3b0f 100644 --- a/docs/vocs/docs/pages/cli/reth/db/diff.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/diff.mdx @@ -193,10 +193,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/drop.mdx b/docs/vocs/docs/pages/cli/reth/db/drop.mdx index ddeb95488d7..5b000b4b70a 100644 --- a/docs/vocs/docs/pages/cli/reth/db/drop.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/drop.mdx @@ -127,10 +127,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/get.mdx b/docs/vocs/docs/pages/cli/reth/db/get.mdx index 7c5252d4363..42d7472534a 100644 --- a/docs/vocs/docs/pages/cli/reth/db/get.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/get.mdx @@ -130,10 +130,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/get/mdbx.mdx b/docs/vocs/docs/pages/cli/reth/db/get/mdbx.mdx index 95604d7e886..1737d1a40da 100644 --- a/docs/vocs/docs/pages/cli/reth/db/get/mdbx.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/get/mdbx.mdx @@ -143,10 +143,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/get/rocksdb.mdx b/docs/vocs/docs/pages/cli/reth/db/get/rocksdb.mdx index 040f5ed4587..18653588067 100644 --- a/docs/vocs/docs/pages/cli/reth/db/get/rocksdb.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/get/rocksdb.mdx @@ -148,10 +148,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/get/static-file.mdx b/docs/vocs/docs/pages/cli/reth/db/get/static-file.mdx index e1de6ed1052..4350583779a 100644 --- a/docs/vocs/docs/pages/cli/reth/db/get/static-file.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/get/static-file.mdx @@ -143,10 +143,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/list.mdx b/docs/vocs/docs/pages/cli/reth/db/list.mdx index f0568b3060f..ef0e901c5de 100644 --- a/docs/vocs/docs/pages/cli/reth/db/list.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/list.mdx @@ -170,10 +170,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/migrate-v2.mdx b/docs/vocs/docs/pages/cli/reth/db/migrate-v2.mdx index c5efcb691b3..d3171f9a455 100644 --- a/docs/vocs/docs/pages/cli/reth/db/migrate-v2.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/migrate-v2.mdx @@ -124,10 +124,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/path.mdx b/docs/vocs/docs/pages/cli/reth/db/path.mdx index 433a510dace..dca6d25adbd 100644 --- a/docs/vocs/docs/pages/cli/reth/db/path.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/path.mdx @@ -124,10 +124,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/prune-checkpoints.mdx b/docs/vocs/docs/pages/cli/reth/db/prune-checkpoints.mdx index a0e8f339f0f..d08ea6efa6e 100644 --- a/docs/vocs/docs/pages/cli/reth/db/prune-checkpoints.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/prune-checkpoints.mdx @@ -129,10 +129,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/prune-checkpoints/get.mdx b/docs/vocs/docs/pages/cli/reth/db/prune-checkpoints/get.mdx index 9e0e40ed014..54b6acd4b40 100644 --- a/docs/vocs/docs/pages/cli/reth/db/prune-checkpoints/get.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/prune-checkpoints/get.mdx @@ -129,10 +129,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/prune-checkpoints/set.mdx b/docs/vocs/docs/pages/cli/reth/db/prune-checkpoints/set.mdx index c060fe98048..447a85c8975 100644 --- a/docs/vocs/docs/pages/cli/reth/db/prune-checkpoints/set.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/prune-checkpoints/set.mdx @@ -146,10 +146,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/repair-trie.mdx b/docs/vocs/docs/pages/cli/reth/db/repair-trie.mdx index 2edf3a4ad3f..feb60cb4386 100644 --- a/docs/vocs/docs/pages/cli/reth/db/repair-trie.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/repair-trie.mdx @@ -10,7 +10,17 @@ Usage: reth db repair-trie [OPTIONS] Options: --dry-run - Only show inconsistencies without making any repairs + Only show inconsistencies without making any repairs. + + Requires the node to be stopped so all workers read the same database snapshot. + + --storage-workers + Number of storage verification workers. Defaults to the number of available CPUs + + --storage-lookahead + Maximum storage roots computed or in flight ahead of the account walk + + [default: 1000000] --metrics Enable Prometheus metrics. @@ -132,10 +142,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/settings.mdx b/docs/vocs/docs/pages/cli/reth/db/settings.mdx index f82cb143e86..e8aa1d93c22 100644 --- a/docs/vocs/docs/pages/cli/reth/db/settings.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/settings.mdx @@ -129,10 +129,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/settings/get.mdx b/docs/vocs/docs/pages/cli/reth/db/settings/get.mdx index 31852a996db..c9181c8eacd 100644 --- a/docs/vocs/docs/pages/cli/reth/db/settings/get.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/settings/get.mdx @@ -124,10 +124,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/settings/set.mdx b/docs/vocs/docs/pages/cli/reth/db/settings/set.mdx index 220ba6ed7a6..8a34a5d960d 100644 --- a/docs/vocs/docs/pages/cli/reth/db/settings/set.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/settings/set.mdx @@ -128,10 +128,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/settings/set/v2.mdx b/docs/vocs/docs/pages/cli/reth/db/settings/set/v2.mdx index 364e21ae391..4135be66d02 100644 --- a/docs/vocs/docs/pages/cli/reth/db/settings/set/v2.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/settings/set/v2.mdx @@ -128,10 +128,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/stage-checkpoints.mdx b/docs/vocs/docs/pages/cli/reth/db/stage-checkpoints.mdx index d7d4da5d24b..af576484967 100644 --- a/docs/vocs/docs/pages/cli/reth/db/stage-checkpoints.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/stage-checkpoints.mdx @@ -129,10 +129,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/stage-checkpoints/get.mdx b/docs/vocs/docs/pages/cli/reth/db/stage-checkpoints/get.mdx index a6715e2af0c..b0821403352 100644 --- a/docs/vocs/docs/pages/cli/reth/db/stage-checkpoints/get.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/stage-checkpoints/get.mdx @@ -129,10 +129,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/stage-checkpoints/set.mdx b/docs/vocs/docs/pages/cli/reth/db/stage-checkpoints/set.mdx index d0fee1da957..2d724d56ba6 100644 --- a/docs/vocs/docs/pages/cli/reth/db/stage-checkpoints/set.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/stage-checkpoints/set.mdx @@ -135,10 +135,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/state.mdx b/docs/vocs/docs/pages/cli/reth/db/state.mdx index 3670692634c..437d7f3b115 100644 --- a/docs/vocs/docs/pages/cli/reth/db/state.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/state.mdx @@ -142,10 +142,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/static-file-header.mdx b/docs/vocs/docs/pages/cli/reth/db/static-file-header.mdx index 508cab81e10..a3874485022 100644 --- a/docs/vocs/docs/pages/cli/reth/db/static-file-header.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/static-file-header.mdx @@ -129,10 +129,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/static-file-header/block.mdx b/docs/vocs/docs/pages/cli/reth/db/static-file-header/block.mdx index 6c009de1388..889cf82dd0b 100644 --- a/docs/vocs/docs/pages/cli/reth/db/static-file-header/block.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/static-file-header/block.mdx @@ -139,10 +139,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/static-file-header/path.mdx b/docs/vocs/docs/pages/cli/reth/db/static-file-header/path.mdx index 44cfaffa058..de2541e4d72 100644 --- a/docs/vocs/docs/pages/cli/reth/db/static-file-header/path.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/static-file-header/path.mdx @@ -128,10 +128,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/stats.mdx b/docs/vocs/docs/pages/cli/reth/db/stats.mdx index d76216f7a61..1f2fc16b3ab 100644 --- a/docs/vocs/docs/pages/cli/reth/db/stats.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/stats.mdx @@ -140,10 +140,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/db/version.mdx b/docs/vocs/docs/pages/cli/reth/db/version.mdx index df2146bc045..9b4fd0daefe 100644 --- a/docs/vocs/docs/pages/cli/reth/db/version.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/version.mdx @@ -124,10 +124,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/download.mdx b/docs/vocs/docs/pages/cli/reth/download.mdx index 37f67fc2cb1..9196b775c80 100644 --- a/docs/vocs/docs/pages/cli/reth/download.mdx +++ b/docs/vocs/docs/pages/cli/reth/download.mdx @@ -347,10 +347,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/dump-genesis.mdx b/docs/vocs/docs/pages/cli/reth/dump-genesis.mdx index 352d698a8f7..5e305012896 100644 --- a/docs/vocs/docs/pages/cli/reth/dump-genesis.mdx +++ b/docs/vocs/docs/pages/cli/reth/dump-genesis.mdx @@ -123,10 +123,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/export-era.mdx b/docs/vocs/docs/pages/cli/reth/export-era.mdx index 87720700d20..94f74710c93 100644 --- a/docs/vocs/docs/pages/cli/reth/export-era.mdx +++ b/docs/vocs/docs/pages/cli/reth/export-era.mdx @@ -261,10 +261,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/import-era.mdx b/docs/vocs/docs/pages/cli/reth/import-era.mdx index 40b00731ba0..44899065b4b 100644 --- a/docs/vocs/docs/pages/cli/reth/import-era.mdx +++ b/docs/vocs/docs/pages/cli/reth/import-era.mdx @@ -260,10 +260,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/import.mdx b/docs/vocs/docs/pages/cli/reth/import.mdx index 1d857873b7e..447bec25f92 100644 --- a/docs/vocs/docs/pages/cli/reth/import.mdx +++ b/docs/vocs/docs/pages/cli/reth/import.mdx @@ -254,10 +254,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/init-state.mdx b/docs/vocs/docs/pages/cli/reth/init-state.mdx index 2134e760563..c12b33f9614 100644 --- a/docs/vocs/docs/pages/cli/reth/init-state.mdx +++ b/docs/vocs/docs/pages/cli/reth/init-state.mdx @@ -267,10 +267,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/init.mdx b/docs/vocs/docs/pages/cli/reth/init.mdx index 4787cc7fbfe..d3cf98224da 100644 --- a/docs/vocs/docs/pages/cli/reth/init.mdx +++ b/docs/vocs/docs/pages/cli/reth/init.mdx @@ -235,10 +235,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index ebdcf93ddf0..ede3fb486f7 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -1389,10 +1389,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/p2p.mdx b/docs/vocs/docs/pages/cli/reth/p2p.mdx index 0033253f963..73920fc2a13 100644 --- a/docs/vocs/docs/pages/cli/reth/p2p.mdx +++ b/docs/vocs/docs/pages/cli/reth/p2p.mdx @@ -122,10 +122,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/p2p/body.mdx b/docs/vocs/docs/pages/cli/reth/p2p/body.mdx index 1758598d62e..327c037907d 100644 --- a/docs/vocs/docs/pages/cli/reth/p2p/body.mdx +++ b/docs/vocs/docs/pages/cli/reth/p2p/body.mdx @@ -396,10 +396,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/p2p/bootnode.mdx b/docs/vocs/docs/pages/cli/reth/p2p/bootnode.mdx index efc85e3bd38..f7eb15b3a93 100644 --- a/docs/vocs/docs/pages/cli/reth/p2p/bootnode.mdx +++ b/docs/vocs/docs/pages/cli/reth/p2p/bootnode.mdx @@ -139,10 +139,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/p2p/enode.mdx b/docs/vocs/docs/pages/cli/reth/p2p/enode.mdx index bdef5c6bd4e..aa32bdbce26 100644 --- a/docs/vocs/docs/pages/cli/reth/p2p/enode.mdx +++ b/docs/vocs/docs/pages/cli/reth/p2p/enode.mdx @@ -128,10 +128,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/p2p/header.mdx b/docs/vocs/docs/pages/cli/reth/p2p/header.mdx index 94cd50070ed..6664b8cb1c8 100644 --- a/docs/vocs/docs/pages/cli/reth/p2p/header.mdx +++ b/docs/vocs/docs/pages/cli/reth/p2p/header.mdx @@ -396,10 +396,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/p2p/rlpx.mdx b/docs/vocs/docs/pages/cli/reth/p2p/rlpx.mdx index 85ec3464f2b..e41a94b85ab 100644 --- a/docs/vocs/docs/pages/cli/reth/p2p/rlpx.mdx +++ b/docs/vocs/docs/pages/cli/reth/p2p/rlpx.mdx @@ -118,10 +118,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/p2p/rlpx/ping.mdx b/docs/vocs/docs/pages/cli/reth/p2p/rlpx/ping.mdx index b2a97b989d5..408ae17835f 100644 --- a/docs/vocs/docs/pages/cli/reth/p2p/rlpx/ping.mdx +++ b/docs/vocs/docs/pages/cli/reth/p2p/rlpx/ping.mdx @@ -118,10 +118,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/prune.mdx b/docs/vocs/docs/pages/cli/reth/prune.mdx index 49dbe38d423..dfd0f3410a2 100644 --- a/docs/vocs/docs/pages/cli/reth/prune.mdx +++ b/docs/vocs/docs/pages/cli/reth/prune.mdx @@ -253,10 +253,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/re-execute.mdx b/docs/vocs/docs/pages/cli/reth/re-execute.mdx index 93472a61c91..a4847610ec3 100644 --- a/docs/vocs/docs/pages/cli/reth/re-execute.mdx +++ b/docs/vocs/docs/pages/cli/reth/re-execute.mdx @@ -296,10 +296,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/snapshot-manifest.mdx b/docs/vocs/docs/pages/cli/reth/snapshot-manifest.mdx index 99c58c88a7f..e9526c8f386 100644 --- a/docs/vocs/docs/pages/cli/reth/snapshot-manifest.mdx +++ b/docs/vocs/docs/pages/cli/reth/snapshot-manifest.mdx @@ -138,10 +138,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/stage.mdx b/docs/vocs/docs/pages/cli/reth/stage.mdx index 06fe000a2d1..2b905b9196f 100644 --- a/docs/vocs/docs/pages/cli/reth/stage.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage.mdx @@ -121,10 +121,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/stage/drop.mdx b/docs/vocs/docs/pages/cli/reth/stage/drop.mdx index efcdaf12946..db1f5f96852 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/drop.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/drop.mdx @@ -249,10 +249,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/stage/dump.mdx b/docs/vocs/docs/pages/cli/reth/stage/dump.mdx index e99abb8aa86..15fe981c9dd 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/dump.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/dump.mdx @@ -242,10 +242,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/stage/dump/account-hashing.mdx b/docs/vocs/docs/pages/cli/reth/stage/dump/account-hashing.mdx index db8236fa811..ae8aa9748a6 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/dump/account-hashing.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/dump/account-hashing.mdx @@ -136,10 +136,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/stage/dump/execution.mdx b/docs/vocs/docs/pages/cli/reth/stage/dump/execution.mdx index 0376005f1b5..8c775b32fa0 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/dump/execution.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/dump/execution.mdx @@ -136,10 +136,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/stage/dump/merkle.mdx b/docs/vocs/docs/pages/cli/reth/stage/dump/merkle.mdx index 600f096ce34..3371910a1f2 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/dump/merkle.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/dump/merkle.mdx @@ -136,10 +136,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/stage/dump/storage-hashing.mdx b/docs/vocs/docs/pages/cli/reth/stage/dump/storage-hashing.mdx index 4b6ae04bb30..f4b1495f60a 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/dump/storage-hashing.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/dump/storage-hashing.mdx @@ -136,10 +136,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/stage/run.mdx b/docs/vocs/docs/pages/cli/reth/stage/run.mdx index 3af2dc85b77..3589dc214ac 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/run.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/run.mdx @@ -521,10 +521,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/stage/unwind.mdx b/docs/vocs/docs/pages/cli/reth/stage/unwind.mdx index db060092df7..7eba44897c5 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/unwind.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/unwind.mdx @@ -243,10 +243,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/stage/unwind/num-blocks.mdx b/docs/vocs/docs/pages/cli/reth/stage/unwind/num-blocks.mdx index 96adab71eee..af7a7faefcd 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/unwind/num-blocks.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/unwind/num-blocks.mdx @@ -128,10 +128,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/docs/pages/cli/reth/stage/unwind/to-block.mdx b/docs/vocs/docs/pages/cli/reth/stage/unwind/to-block.mdx index a2c7134e2e2..33f3f8a65c4 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/unwind/to-block.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/unwind/to-block.mdx @@ -128,10 +128,12 @@ Display: Tracing: --tracing-otlp[=] - Enable `Opentelemetry` tracing export to an OTLP endpoint. + Enable `Opentelemetry` tracing export to an OTLP endpoint or local file. If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + A `file:///absolute/path` endpoint appends length-delimited OTLP protobuf batches without a collector. Use a separate file per process. The network protocol is ignored for files. + Example: --tracing-otlp=http://collector:4318/v1/traces [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] diff --git a/docs/vocs/scripts/build-cargo-docs.sh b/docs/vocs/scripts/build-cargo-docs.sh index 634f4acf77b..e03c324e779 100755 --- a/docs/vocs/scripts/build-cargo-docs.sh +++ b/docs/vocs/scripts/build-cargo-docs.sh @@ -10,6 +10,6 @@ echo "Building cargo docs..." # Build the documentation export RUSTDOCFLAGS="--cfg docsrs --show-type-layout --generate-link-to-definition --enable-index-page -Zunstable-options" -cargo +nightly docs --exclude "example-*" +cargo +"${RUST_NIGHTLY:-nightly}" docs --exclude "example-*" echo "Cargo docs built successfully at ./target/doc" diff --git a/docs/vocs/vocs.config.ts b/docs/vocs/vocs.config.ts index df2051eb973..b53d91bd759 100644 --- a/docs/vocs/vocs.config.ts +++ b/docs/vocs/vocs.config.ts @@ -23,7 +23,7 @@ export default defineConfig({ { text: 'Rustdocs', link: '/docs' }, { text: 'GitHub', link: 'https://github.com/paradigmxyz/reth' }, { - text: 'v2.9.0', + text: 'v2.10.0', items: [ { text: 'Releases', diff --git a/examples/custom-evm/src/main.rs b/examples/custom-evm/src/main.rs index f2285fdea56..27f1188dfc1 100644 --- a/examples/custom-evm/src/main.rs +++ b/examples/custom-evm/src/main.rs @@ -46,7 +46,7 @@ impl EvmFactory for MyEvmFactory { fn precompiles(&self, spec: SpecId) -> evm2::Precompiles { let mut precompiles = evm2::Precompiles::base(spec); - if spec == SpecId::PRAGUE { + if spec.enables(SpecId::PRAGUE) { precompiles.as_map_mut().insert(Precompile::new( address!("0x0000000000000000000000000000000000000999"), PrecompileId::custom("custom"), diff --git a/scripts/bench-reth-charts/uv.lock b/scripts/bench-reth-charts/uv.lock index 08cf1a4d692..df59189f66e 100644 --- a/scripts/bench-reth-charts/uv.lock +++ b/scripts/bench-reth-charts/uv.lock @@ -981,14 +981,15 @@ wheels = [ [[package]] name = "python-dateutil" -version = "2.9.0.post0" +version = "3.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "six" }, + { name = "tzdata" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0e/df/f15f80383f152452d8985b36d95cb8751f901a3e0ff4cd27f7e118f168e2/python_dateutil-3.0.0.tar.gz", hash = "sha256:7ebcb3c29a92f3ceafe79a7a48a5c45677e9e004bbeec5742ac0bbb00560b7a2", size = 271066, upload-time = "2026-10-21T23:41:45.075Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, + { url = "https://files.pythonhosted.org/packages/5a/17/cebee01d1c0c1441174cf0456ddc91b280d79acb65340c05e41d77866ee8/python_dateutil-3.0.0-py2.py3-none-any.whl", hash = "sha256:3c79630202e4152c6add678446984236efe4f57e84d8e892aab0637bab82e9fd", size = 94659, upload-time = "2026-10-21T23:41:43.431Z" }, ] [[package]] @@ -999,3 +1000,12 @@ sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68 wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] + +[[package]] +name = "tzdata" +version = "2026.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7a/29/6a1391d31cd00ff9e927215a224a880e1e84e826b64ba80a499e51c074d8/tzdata-2026.6.tar.gz", hash = "sha256:85d2cd9c09f81f0bb29db27759fc83af4f1e620efb814f8c5bf8b4cc4feece6d", size = 201017, upload-time = "2026-10-28T09:57:09.137Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/66/9a06c2227fedfd4919f0eaaa8b94377d9730882a7e4f95823c9a331ab5e1/tzdata-2026.6-py2.py3-none-any.whl", hash = "sha256:0a10eec6e13ef16b627fabed9a5f2ba61a62b1c67f327305f4e7e412a70b4aec", size = 348210, upload-time = "2026-10-28T09:57:07.276Z" }, +] diff --git a/testing/testing-utils/src/generators.rs b/testing/testing-utils/src/generators.rs index 152dcfb84e8..b3e56388692 100644 --- a/testing/testing-utils/src/generators.rs +++ b/testing/testing-utils/src/generators.rs @@ -10,7 +10,7 @@ use alloy_eips::{ }; use alloy_primitives::{Address, BlockNumber, Bytes, TxKind, B256, B64, U256}; pub use rand::Rng; -use rand::{distr::uniform::SampleRange, rngs::StdRng, SeedableRng}; +use rand::{distr::uniform::SampleRange, prelude::IndexedRandom, rngs::StdRng, SeedableRng}; use reth_ethereum_primitives::{Block, BlockBody, Receipt, Transaction, TransactionSigned}; use reth_primitives_traits::{ crypto::secp256k1::sign_message, proofs, Account, Block as _, Log, SealedBlock, SealedHeader, @@ -384,7 +384,6 @@ pub fn random_account_change( n_storage_changes: Range, key_range: Range, ) -> (Address, Address, U256, Vec) { - use rand::prelude::IndexedRandom; let mut addresses = valid_addresses.choose_multiple(rng, 2).copied(); let addr_from = addresses.next().unwrap_or_else(Address::random);