diff --git a/.circleci/config.yml b/.circleci/config.yml index 4ed70d500ab..883610aeae9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -45,6 +45,23 @@ jobs: name: cargo deny check command: cargo deny --log-level warn --all-features check all + compression-compatibility: + docker: + - image: rust:1.95-bookworm@sha256:6258907abe69656e41cd992e0b705cdcfabcbbe3db374f92ed2d47121282d4a1 + resource_class: medium + steps: + - checkout + - run: + name: Install Clippy + command: rustup component add clippy + - run: + name: Check compression compatibility + command: cargo test --locked --manifest-path .github/compatibility/compression/Cargo.toml + - run: + name: Lint compression compatibility tests + command: cargo clippy --locked --manifest-path .github/compatibility/compression/Cargo.toml --all-targets -- -D warnings + + prepare-act-event: machine: image: ubuntu-2404:2025.09.1 @@ -227,6 +244,7 @@ workflows: - feature-propagation - cargo-deny: name: lint-deny + - compression-compatibility - act: wf: unit.yml concurrent_jobs: 1 diff --git a/.github/compatibility/compression/Cargo.lock b/.github/compatibility/compression/Cargo.lock new file mode 100644 index 00000000000..5de2598c407 --- /dev/null +++ b/.github/compatibility/compression/Cargo.lock @@ -0,0 +1,437 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "cc" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-timer" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-macro", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom", + "libc", +] + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + +[[package]] +name = "reth-compression-compatibility-tests" +version = "0.0.0" +dependencies = [ + "reth-zstd-compressors 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "reth-zstd-compressors 0.7.1 (git+https://github.com/ethereum-optimism/reth-core?rev=46719f79ff5f1b94bae3738cd65a4e44774b8e09)", + "rstest", + "zstd 0.13.3", + "zstd 0.14.3", + "zstd-safe 7.2.4", +] + +[[package]] +name = "reth-zstd-compressors" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbdee2f0ad09dc32b43ab52641f8239766a66de4345834bad27d5cfa2f62864d" +dependencies = [ + "zstd 0.13.3", +] + +[[package]] +name = "reth-zstd-compressors" +version = "0.7.1" +source = "git+https://github.com/ethereum-optimism/reth-core?rev=46719f79ff5f1b94bae3738cd65a4e44774b8e09#46719f79ff5f1b94bae3738cd65a4e44774b8e09" +dependencies = [ + "zstd 0.14.3", +] + +[[package]] +name = "rstest" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49" +dependencies = [ + "futures-timer", + "futures-util", + "rstest_macros", +] + +[[package]] +name = "rstest_macros" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn 2.0.119", + "unicode-ident", +] + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "toml_datetime" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.17+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3641d5bbb5349a79e1020a242d251efbc546ad8048d133958323ce9c40a9c9c" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" +dependencies = [ + "winnow", +] + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe 7.2.4", +] + +[[package]] +name = "zstd" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70" +dependencies = [ + "zstd-safe 8.2.0", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-safe" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded520f16b29e1a9b771c722221b2b0a0398c2316d9bb3c42b458a5414d7434b" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.1.2+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/.github/compatibility/compression/Cargo.toml b/.github/compatibility/compression/Cargo.toml new file mode 100644 index 00000000000..4c9103aaae9 --- /dev/null +++ b/.github/compatibility/compression/Cargo.toml @@ -0,0 +1,16 @@ +[package] +edition = "2024" +license = "MIT OR Apache-2.0" +name = "reth-compression-compatibility-tests" +publish = false +version = "0.0.0" + +[workspace] + +[dev-dependencies] +new-compressors = { git = "https://github.com/ethereum-optimism/reth-core", package = "reth-zstd-compressors", rev = "46719f79ff5f1b94bae3738cd65a4e44774b8e09" } +new-zstd = { package = "zstd", version = "=0.14.3" } +old-compressors = { package = "reth-zstd-compressors", version = "=0.7.1" } +old-zstd = { package = "zstd", version = "=0.13.3" } +old-zstd-safe = { package = "zstd-safe", version = "=7.2.4" } +rstest = "0.26.1" diff --git a/.github/compatibility/compression/tests/compatibility.rs b/.github/compatibility/compression/tests/compatibility.rs new file mode 100644 index 00000000000..98f5ae45e1d --- /dev/null +++ b/.github/compatibility/compression/tests/compatibility.rs @@ -0,0 +1,77 @@ +use rstest::rstest; + +#[derive(Clone, Copy, Debug)] +enum CompressionFormat { + NippyJar, + NippyJarDictionary, + Receipt, + Transaction, +} + +mod previous { + use old_compressors as compressors; + use old_zstd as zstd; + + include!("support/codec.rs"); +} + +mod updated { + use new_compressors as compressors; + use new_zstd as zstd; + + include!("support/codec.rs"); +} + +#[rstest] +#[case::nippy_jar(CompressionFormat::NippyJar)] +#[case::nippy_jar_dictionary(CompressionFormat::NippyJarDictionary)] +#[case::receipt(CompressionFormat::Receipt)] +#[case::transaction(CompressionFormat::Transaction)] +fn updated_reader_accepts_previous_frames( + #[case] format: CompressionFormat, + #[values(0, 256, 4096, 16384)] data_size: usize, +) -> std::io::Result<()> { + let original = test_payload(data_size); + let compressed = previous::encode(format, &original)?; + + assert_eq!(updated::decode(format, &compressed, original.len())?, original); + Ok(()) +} + +#[rstest] +#[case::nippy_jar(CompressionFormat::NippyJar)] +#[case::nippy_jar_dictionary(CompressionFormat::NippyJarDictionary)] +#[case::receipt(CompressionFormat::Receipt)] +#[case::transaction(CompressionFormat::Transaction)] +fn previous_reader_accepts_updated_frames( + #[case] format: CompressionFormat, + #[values(0, 256, 4096, 16384)] data_size: usize, +) -> std::io::Result<()> { + let original = test_payload(data_size); + let compressed = updated::encode(format, &original)?; + + assert_eq!(previous::decode(format, &compressed, original.len())?, original); + Ok(()) +} + +#[rstest] +#[case::nippy_jar(CompressionFormat::NippyJar)] +#[case::nippy_jar_dictionary(CompressionFormat::NippyJarDictionary)] +fn updated_reader_rejects_malformed_frames(#[case] format: CompressionFormat) { + assert!(updated::decode(format, b"invalid zstd frame", 16384).is_err()); +} + +#[test] +fn compression_dictionaries_are_unchanged() { + assert_eq!(new_compressors::RECEIPT_DICTIONARY, old_compressors::RECEIPT_DICTIONARY); + assert_eq!(new_compressors::TRANSACTION_DICTIONARY, old_compressors::TRANSACTION_DICTIONARY); +} + +#[test] +fn native_library_matches_release_version() { + assert_eq!(old_zstd_safe::version_number(), 10507); +} + +fn test_payload(data_size: usize) -> Vec { + (0u8..=255).cycle().take(data_size).collect() +} diff --git a/.github/compatibility/compression/tests/support/codec.rs b/.github/compatibility/compression/tests/support/codec.rs new file mode 100644 index 00000000000..81b22760a97 --- /dev/null +++ b/.github/compatibility/compression/tests/support/codec.rs @@ -0,0 +1,36 @@ +use super::CompressionFormat; + +pub(super) fn encode(format: CompressionFormat, data: &[u8]) -> std::io::Result> { + match format { + CompressionFormat::NippyJar => zstd::stream::encode_all(data, 0), + CompressionFormat::NippyJarDictionary => { + let dictionary = + zstd::dict::EncoderDictionary::copy(compressors::TRANSACTION_DICTIONARY, 0); + zstd::bulk::Compressor::with_prepared_dictionary(&dictionary)?.compress(data) + } + CompressionFormat::Receipt => compressors::create_receipt_compressor().compress(data), + CompressionFormat::Transaction => compressors::create_tx_compressor().compress(data), + } +} + +pub(super) fn decode( + format: CompressionFormat, + frame: &[u8], + data_size: usize, +) -> std::io::Result> { + match format { + CompressionFormat::NippyJar => zstd::stream::decode_all(frame), + CompressionFormat::NippyJarDictionary => { + let dictionary = + zstd::dict::DecoderDictionary::copy(compressors::TRANSACTION_DICTIONARY); + zstd::bulk::Decompressor::with_prepared_dictionary(&dictionary)? + .decompress(frame, data_size) + } + CompressionFormat::Receipt => { + Ok(compressors::create_receipt_decompressor().decompress(frame).to_vec()) + } + CompressionFormat::Transaction => { + Ok(compressors::create_tx_decompressor().decompress(frame).to_vec()) + } + } +} diff --git a/.github/scripts/compact-base.sh b/.github/scripts/compact-base.sh new file mode 100644 index 00000000000..651791910e1 --- /dev/null +++ b/.github/scripts/compact-base.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ -n "${BASE_REF:-}" ]]; then + printf '%s\n' "$BASE_REF" + exit 0 +fi + +case "${REF:?missing workflow ref}" in + refs/heads/op-v*) + printf '%s\n' "${REF#refs/heads/op-}" + ;; + refs/heads/op-*) + echo "no compact base for ${REF}: only op-v branches map to an upstream tag" >&2 + exit 1 + ;; + *) + if [[ "${REPOSITORY:?missing workflow repository}" == "ethereum-optimism/reth" ]]; then + release_version="$(awk -F '"' '/^\[workspace.package\]/{package_section=1;next} package_section && /^\[/{exit} package_section && /^version *=/{print $2;exit}' Cargo.toml)" + if [[ ! "$release_version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "cannot select a compact release base for workspace version ${release_version}" >&2 + exit 1 + fi + printf 'v%s\n' "$release_version" + else + printf 'main\n' + fi + ;; +esac diff --git a/.github/scripts/compact-clean-source.sh b/.github/scripts/compact-clean-source.sh new file mode 100644 index 00000000000..6f42901546b --- /dev/null +++ b/.github/scripts/compact-clean-source.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +tracked_paths_file="$(mktemp)" +trap 'rm -f -- "$tracked_paths_file"' EXIT +git ls-files -z > "$tracked_paths_file" + +# act overlays checkouts, so remove tracked base files that the candidate may no longer contain. +while IFS= read -r -d '' tracked_path; do + case "$tracked_path" in + testdata/micro/compact/*) continue ;; + esac + if [[ -f "$tracked_path" || -L "$tracked_path" ]]; then + rm -f -- "$tracked_path" + fi +done < "$tracked_paths_file" diff --git a/.github/scripts/test_compact_workflow.py b/.github/scripts/test_compact_workflow.py new file mode 100644 index 00000000000..0bc35e2af3f --- /dev/null +++ b/.github/scripts/test_compact_workflow.py @@ -0,0 +1,101 @@ +import os +from pathlib import Path +import subprocess +import shutil +import textwrap +import tempfile +import unittest + +SCRIPT_DIRECTORY = Path(__file__).resolve().parent + + +class CompactWorkflowTests(unittest.TestCase): + def test_comparison_base(self): + cases = [ + ("op-v2.6.0", "refs/heads/topic", "ethereum-optimism/reth", "2.6.0", "op-v2.6.0"), + ("", "refs/heads/op-v2.6.0", "ethereum-optimism/reth", "2.6.0", "v2.6.0"), + ("", "refs/heads/topic", "ethereum-optimism/reth", "2.6.0", "v2.6.0"), + ("", "refs/heads/topic", "paradigmxyz/reth", "2.8.0", "main"), + ] + for base_ref, workflow_ref, repository, version, expected in cases: + with self.subTest(base_ref=base_ref, repository=repository, workflow_ref=workflow_ref): + with tempfile.TemporaryDirectory() as workspace: + Path(workspace, "Cargo.toml").write_text(f'[workspace.package]\nversion = "{version}"\n') + environment = dict(os.environ, BASE_REF=base_ref, REF=workflow_ref, REPOSITORY=repository) + result = subprocess.run( + ["bash", str(SCRIPT_DIRECTORY / "compact-base.sh")], + capture_output=True, + cwd=workspace, + env=environment, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), expected) + + def test_unmapped_release_fails(self): + cases = [("refs/heads/op-topic", "2.6.0"), ("refs/heads/topic", "2.6.0-dev")] + for workflow_ref, version in cases: + with self.subTest(version=version, workflow_ref=workflow_ref): + with tempfile.TemporaryDirectory() as workspace: + Path(workspace, "Cargo.toml").write_text(f'[workspace.package]\nversion = "{version}"\n') + environment = dict(os.environ, BASE_REF="", REF=workflow_ref, REPOSITORY="ethereum-optimism/reth") + result = subprocess.run( + ["bash", str(SCRIPT_DIRECTORY / "compact-base.sh")], + capture_output=True, + cwd=workspace, + env=environment, + text=True, + ) + self.assertNotEqual(result.returncode, 0) + + def test_invalid_release_fails_workflow_step(self): + workflow = (SCRIPT_DIRECTORY.parent / "workflows" / "compact.yml").read_text() + select_step = workflow.split(" - name: Select base\n", 1)[1].split(" - name: Checkout base", 1)[0] + command = textwrap.dedent(select_step.split(" run: |\n", 1)[1]) + with tempfile.TemporaryDirectory() as workspace: + root = Path(workspace) + scripts = root / ".github" / "scripts" + scripts.mkdir(parents=True) + for name in ["compact-base.sh", "compact-clean-source.sh"]: + shutil.copyfile(SCRIPT_DIRECTORY / name, scripts / name) + (root / "Cargo.toml").write_text('[workspace.package]\nversion = "2.6.0-dev"\n') + output_path = root / "github-output" + environment = dict(os.environ, BASE_REF="", REF="refs/heads/topic", REPOSITORY="ethereum-optimism/reth", GITHUB_OUTPUT=str(output_path), RUNNER_TEMP=str(root)) + result = subprocess.run(["bash", "-e", "-c", command], capture_output=True, cwd=root, env=environment, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(output_path.exists()) + self.assertFalse((root / "compact-clean-source.sh").exists()) + + def test_cleanup_requires_a_checkout(self): + with tempfile.TemporaryDirectory() as workspace: + result = subprocess.run( + ["bash", str(SCRIPT_DIRECTORY / "compact-clean-source.sh")], + capture_output=True, + cwd=workspace, + text=True, + ) + self.assertNotEqual(result.returncode, 0) + + def test_source_cleanup_preserves_vectors_and_build_cache(self): + with tempfile.TemporaryDirectory() as workspace: + root = Path(workspace) + subprocess.run(["git", "init", "--quiet"], check=True, cwd=root) + stale_source = root / "proof_task" / "mod.rs" + stale_source.parent.mkdir() + stale_source.write_text("stale module") + vectors = root / "testdata" / "micro" / "compact" / "Header.json" + vectors.parent.mkdir(parents=True) + vectors.write_text("generated vectors") + subprocess.run(["git", "add", "."], check=True, cwd=root) + build_cache = root / "target" / "cached-artifact" + build_cache.parent.mkdir() + build_cache.write_text("cached build") + subprocess.run(["bash", str(SCRIPT_DIRECTORY / "compact-clean-source.sh")], check=True, cwd=root) + self.assertFalse(stale_source.exists()) + self.assertEqual(vectors.read_text(), "generated vectors") + self.assertEqual(build_cache.read_text(), "cached build") + self.assertTrue((root / ".git").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/compact.yml b/.github/workflows/compact.yml index 86e2fb8d9a2..f52c109c4b6 100644 --- a/.github/workflows/compact.yml +++ b/.github/workflows/compact.yml @@ -41,28 +41,18 @@ jobs: - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true - # An `op-v` branch builds on upstream tag `v`. Pushes to it must decode - # that release's data, not unreleased main formats. + - name: Test compact workflow helpers + run: python3 .github/scripts/test_compact_workflow.py - name: Select base id: base env: BASE_REF: ${{ github.base_ref }} REF: ${{ github.ref }} + REPOSITORY: ${{ github.repository }} run: | - set -euo pipefail - if [ -n "$BASE_REF" ]; then - base="$BASE_REF" - else - case "$REF" in - refs/heads/op-v*) base="${REF#refs/heads/op-}" ;; - refs/heads/op-*) - echo "no compact base for ${REF}: only op-v branches map to an upstream tag" >&2 - exit 1 - ;; - *) base=main ;; - esac - fi - echo "ref=$base" >> "$GITHUB_OUTPUT" + comparison_base="$(bash .github/scripts/compact-base.sh)" + printf 'ref=%s\n' "$comparison_base" >> "$GITHUB_OUTPUT" + cp .github/scripts/compact-clean-source.sh "$RUNNER_TEMP/compact-clean-source.sh" - name: Checkout base uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -72,6 +62,8 @@ jobs: - name: Generate compact vectors run: | ${{ matrix.bin }} -- test-vectors compact --write + - name: Remove comparison source files + run: bash "$RUNNER_TEMP/compact-clean-source.sh" - name: Checkout PR uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/Cargo.lock b/Cargo.lock index 0af8e7f9822..ea079f05ecd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2511,9 +2511,9 @@ dependencies = [ [[package]] name = "compression-codecs" -version = "0.4.39" +version = "0.4.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fe67f2944eef52fc7b106b8c9450d243a88701a0c065f7f57235e76abaed7df" +checksum = "5f018b36ac50598f7bfb0aa6094281e253e6b3cbc507a69652d2fb963a326d8d" dependencies = [ "brotli", "compression-core", @@ -3011,7 +3011,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab67060fc6b8ef687992d439ca0fa36e7ed17e9a0b16b25b601e8757df720de" dependencies = [ "data-encoding", - "syn 2.0.118", + "syn 1.0.109", ] [[package]] @@ -5853,9 +5853,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.28" +version = "1.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc3a226e576f50782b3305c5ccf458698f92798987f551c6a02efe8276721e22" +checksum = "ff20308ecabd635a2dc1dd5834a86c462cca20efba1d5d3ddab1f8aea3455cb2" dependencies = [ "cc", "libc", @@ -10972,8 +10972,7 @@ dependencies = [ [[package]] name = "reth-zstd-compressors" version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbdee2f0ad09dc32b43ab52641f8239766a66de4345834bad27d5cfa2f62864d" +source = "git+https://github.com/ethereum-optimism/reth-core?rev=46719f79ff5f1b94bae3738cd65a4e44774b8e09#46719f79ff5f1b94bae3738cd65a4e44774b8e09" dependencies = [ "zstd", ] @@ -11634,7 +11633,7 @@ dependencies = [ "rustls-webpki", "security-framework", "security-framework-sys", - "webpki-root-certs 1.0.7", + "webpki-root-certs 1.0.10", "windows-sys 0.59.0", ] @@ -11646,9 +11645,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "aws-lc-rs", "ring", @@ -12795,9 +12794,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.53.1" +version = "1.53.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +checksum = "ce3335fa71841cda333a58d7615b03901380ecf09d59b3296d21f8bbac0dde4e" dependencies = [ "bytes", "libc", @@ -13724,14 +13723,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75c7f0ef91146ebfb530314f5f1d24528d7f0767efbfd31dce919275413e393e" dependencies = [ - "webpki-root-certs 1.0.7", + "webpki-root-certs 1.0.10", ] [[package]] name = "webpki-root-certs" -version = "1.0.7" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" +checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb" dependencies = [ "rustls-pki-types", ] @@ -13742,14 +13741,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.7", + "webpki-roots 1.0.10", ] [[package]] name = "webpki-roots" -version = "1.0.7" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9" dependencies = [ "rustls-pki-types", ] @@ -14482,27 +14481,27 @@ checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" [[package]] name = "zstd" -version = "0.13.3" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.2.4" +version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +checksum = "ded520f16b29e1a9b771c722221b2b0a0398c2316d9bb3c42b458a5414d7434b" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" +version = "2.1.2+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 1b1be1a7134..0c749e82dfe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -152,7 +152,7 @@ members = [ "crates/tracing-otlp", ] default-members = ["bin/reth"] -exclude = ["docs/cli"] +exclude = [".github/compatibility/compression", "docs/cli"] # Explicitly set the resolver to version 2, which is the default for packages with edition >= 2021 # https://doc.rust-lang.org/edition-guide/rust-2021/default-cargo-resolver.html @@ -521,7 +521,7 @@ tar = "0.4.46" tracing = { version = "0.1.0", default-features = false, features = ["attributes"] } tracing-appender = "0.2.5" url = { version = "2.3", default-features = false } -zstd = "0.13" +zstd = "0.14.3" byteorder = "1" fixed-cache = { version = "0.1.10", default-features = false, features = ["alloc"] } moka = "0.12" @@ -661,3 +661,6 @@ vergen-git2 = "10.0.1" # networking ipnet = "2.11" + +[patch.crates-io] +reth-zstd-compressors = { git = "https://github.com/ethereum-optimism/reth-core", rev = "46719f79ff5f1b94bae3738cd65a4e44774b8e09" } diff --git a/crates/ethereum/node/tests/e2e/blobs.rs b/crates/ethereum/node/tests/e2e/blobs.rs index 830cc8befc8..3212040b1bb 100644 --- a/crates/ethereum/node/tests/e2e/blobs.rs +++ b/crates/ethereum/node/tests/e2e/blobs.rs @@ -1,5 +1,5 @@ use crate::utils::eth_payload_attributes; -use alloy_eips::Decodable2718; +use alloy_eips::{merge::SLOT_DURATION_SECS, Decodable2718}; use alloy_genesis::Genesis; use reth_chainspec::{ChainSpecBuilder, MAINNET}; use reth_e2e_test_utils::{ @@ -12,10 +12,7 @@ use reth_node_core::{args::RpcServerArgs, node_config::NodeConfig}; use reth_node_ethereum::EthereumNode; use reth_tasks::Runtime; use reth_transaction_pool::TransactionPool; -use std::{ - sync::Arc, - time::{Duration, SystemTime, UNIX_EPOCH}, -}; +use std::{sync::Arc, time::Duration}; #[tokio::test] async fn can_handle_blobs() -> eyre::Result<()> { @@ -146,9 +143,9 @@ async fn blob_conversion_at_osaka() -> eyre::Result<()> { reth_tracing::init_test_tracing(); let runtime = Runtime::test(); - let current_timestamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(); - // Osaka activates in 2 slots - let osaka_timestamp = current_timestamp + 24; + // Keep genesis outside the conversion window until both legacy sidecars are checked. + let prague_timestamp = 2 * SLOT_DURATION_SECS; + let osaka_timestamp = prague_timestamp + 2 * SLOT_DURATION_SECS; let genesis: Genesis = serde_json::from_str(include_str!("../assets/genesis.json")).unwrap(); let chain_spec = Arc::new( @@ -174,16 +171,10 @@ async fn blob_conversion_at_osaka() -> eyre::Result<()> { let mut node = NodeTestContext::new(node, eth_payload_attributes).await?; - let mut wallets = Wallet::new(3).wallet_gen(); + let mut wallets = Wallet::new(2).wallet_gen(); let first = wallets.pop().unwrap(); let second = wallets.pop().unwrap(); - // build a dummy payload at `current_timestamp` - let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallets.pop().unwrap()).await; - node.rpc.inject_tx(raw_tx).await?; - node.payload.timestamp = current_timestamp - 1; - node.advance_block().await?; - // build blob txs let first_blob = TransactionTestContext::tx_with_blobs_bytes(1, first.clone()).await?; let second_blob = TransactionTestContext::tx_with_blobs_bytes(1, second.clone()).await?; @@ -214,8 +205,10 @@ async fn blob_conversion_at_osaka() -> eyre::Result<()> { TransactionTestContext::validate_sidecar(envelope); // build last Prague payload - node.payload.timestamp = current_timestamp + 1; + node.payload.timestamp = prague_timestamp - 1; let prague_payload = node.new_payload().await?; + assert_eq!(prague_payload.block().timestamp, prague_timestamp); + assert!(prague_payload.block().body().transactions().any(|tx| *tx.hash() == blob_tx_hash)); assert!(matches!(prague_payload.sidecars(), BlobSidecars::Eip4844(_))); // inject second blob tx to the pool @@ -227,7 +220,14 @@ async fn blob_conversion_at_osaka() -> eyre::Result<()> { // validate sidecar TransactionTestContext::validate_sidecar(envelope); - tokio::time::sleep(Duration::from_secs(6)).await; + node.update_forkchoice(genesis_hash, node.submit_payload(prague_payload).await?).await?; + let sidecar_conversion = async { + while !node.inner.pool.get_blob(blob_tx_hash)?.is_some_and(|sidecar| sidecar.is_eip7594()) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + Ok::<(), eyre::Report>(()) + }; + tokio::time::timeout(Duration::from_secs(30), sidecar_conversion).await??; // fetch second blob tx from rpc again let envelope = node.rpc.envelope_by_hash(blob_tx_hash).await?; @@ -236,9 +236,6 @@ async fn blob_conversion_at_osaka() -> eyre::Result<()> { // validate sidecar TransactionTestContext::validate_sidecar(envelope); - // submit the Prague payload - node.update_forkchoice(genesis_hash, node.submit_payload(prague_payload).await?).await?; - // Build first Osaka payload node.payload.timestamp = osaka_timestamp - 1; let osaka_payload = node.new_payload().await?; diff --git a/deny.toml b/deny.toml index be45b22db44..0c64fb81aab 100644 --- a/deny.toml +++ b/deny.toml @@ -12,6 +12,8 @@ ignore = [ "RUSTSEC-2023-0089", # https://rustsec.org/advisories/RUSTSEC-2026-0173 proc-macro-error2 is unmaintained "RUSTSEC-2026-0173", + # crypto-bigint 0.5 has no patched release compatible with elliptic-curve 0.13. + "RUSTSEC-2026-0387", ] # This section is considered when running `cargo deny check bans`. @@ -91,6 +93,7 @@ allow-git = [ "https://github.com/alloy-rs/alloy", "https://github.com/foundry-rs/block-explorers", "https://github.com/bluealloy/revm", + "https://github.com/ethereum-optimism/reth-core", "https://github.com/paradigmxyz/revm-inspectors", "https://github.com/alloy-rs/evm", "https://github.com/alloy-rs/hardforks",