Download splitted genesis

download-utils/src/lib.rs

@@ -7,7 +7,10 @@ use {
         snapshot_package::SnapshotType,
         snapshot_utils::{self, ArchiveFormat},
     },
-    solana_sdk::{clock::Slot, genesis_config::DEFAULT_GENESIS_ARCHIVE, hash::Hash},
+    solana_sdk::{
+        clock::Slot, genesis_config::{DEFAULT_GENESIS_ARCHIVE, EVM_GENESIS_ARCHIVE}, 
+        hash::Hash
+    },
     std::{
         fs::{self, File},
         io::{self, Read},
@@ -231,20 +234,28 @@ pub fn download_genesis_if_missing(
     rpc_addr: &SocketAddr,
     genesis_package: &Path,
     use_progress_bar: bool,
-) -> Result<PathBuf, String> {
+) -> Result<(PathBuf, PathBuf), String> {
     if !genesis_package.exists() {
-        let tmp_genesis_path = genesis_package.parent().unwrap().join("tmp-genesis");
-        let tmp_genesis_package = tmp_genesis_path.join(DEFAULT_GENESIS_ARCHIVE);
+        let tmp_genesis_path = genesis_package.join("tmp-genesis");
+        let tmp_genesis_native = tmp_genesis_path.join(DEFAULT_GENESIS_ARCHIVE);
+        let tmp_genesis_evm = tmp_genesis_path.join(EVM_GENESIS_ARCHIVE);
 
         let _ignored = fs::remove_dir_all(&tmp_genesis_path);
         download_file(
             &format!("http://{}/{}", rpc_addr, DEFAULT_GENESIS_ARCHIVE),
-            &tmp_genesis_package,
+            &tmp_genesis_native,
+            use_progress_bar,
+            &mut None,
+        )?;
+
+        download_file(
+            &format!("http://{}/{}", rpc_addr, EVM_GENESIS_ARCHIVE),
+            &tmp_genesis_evm,
             use_progress_bar,
             &mut None,
         )?;
 
-        Ok(tmp_genesis_package)
+        Ok((tmp_genesis_native, tmp_genesis_evm))
     } else {
         Err("genesis already exists".to_string())
     }

evm-utils/evm-state/src/lib.rs

@@ -23,9 +23,11 @@ pub use {
     context::{ChainContext, EvmConfig},
     state::{
         AccountProvider, ChangedState, Committed, EvmBackend, EvmPersistState, EvmState, Incomming,
-        BURN_GAS_PRICE, DEFAULT_GAS_LIMIT, MAX_IN_MEMORY_EVM_ACCOUNTS,
+        BURN_GAS_PRICE, DEFAULT_GAS_LIMIT, MAX_IN_HEAP_EVM_ACCOUNTS_BYTES,
+        MAX_IN_MEMORY_EVM_ACCOUNTS,
     },
-    storage::Storage, storage::StorageSecondary
+    storage::Storage,
+    storage::StorageSecondary,
 };
 
 pub use executor::{

evm-utils/evm-state/src/state.rs

@@ -24,6 +24,8 @@ pub const DEFAULT_GAS_LIMIT: u64 = 300_000_000;
 pub const BURN_GAS_PRICE: u64 = 2_000_000_000; // 2 lamports per gas.
 /// Dont load to many account to memory, to avoid OOM.
 pub const MAX_IN_MEMORY_EVM_ACCOUNTS: usize = 10000;
+/// Approximate size, real size could be twice as much
+pub const MAX_IN_HEAP_EVM_ACCOUNTS_BYTES: usize = 100_000_000;
 
 pub type ChangedState = HashMap<H160, (Maybe<AccountState>, HashMap<H256, H256>)>;
 
@@ -66,6 +68,13 @@ pub struct Incomming {
 }
 
 impl Incomming {
+    pub fn genesis_from_state(state_root: H256) -> Self {
+        Self {
+            state_root,
+            ..Default::default()
+        }
+    }
+
     fn new(
         block_number: BlockNum,
         state_root: H256,
@@ -542,6 +551,7 @@ impl Default for EvmPersistState {
 }
 
 impl EvmState {
+    /// Clears content of `path` directory and creates new empty `EvmState`
     pub fn new<P: AsRef<Path>>(path: P) -> Result<Self, anyhow::Error> {
         let evm_state = path.as_ref();
         if evm_state.is_dir() && evm_state.exists() {
@@ -553,6 +563,7 @@ impl EvmState {
         Self::load_from(evm_state, Incomming::default(), true)
     }
 
+    /// Clears content of `evm_state` directory and creates new `EvmState` from genesis
     pub fn new_from_genesis(
         evm_state: impl AsRef<Path>,
         evm_genesis: impl AsRef<Path>,

evm-utils/evm-state/src/storage/mod.rs

@@ -1,7 +1,7 @@
 use std::{
     array::TryFromSliceError,
     borrow::Borrow,
-    collections::BTreeSet,
+    collections::{BTreeSet, HashMap},
     convert::TryInto,
     fs,
     io::Error as IoError,
@@ -11,6 +11,7 @@ use std::{
 
 use bincode::config::{BigEndian, DefaultOptions, Options as _, WithOtherEndian};
 use derive_more::{AsRef, Deref};
+use itertools::Itertools;
 use lazy_static::lazy_static;
 use log::*;
 use rlp::{Decodable, Encodable};
@@ -42,6 +43,7 @@ pub use rocksdb; // avoid mess with dependencies for another crates
 
 type DB = OptimisticTransactionDB;
 type BincodeOpts = WithOtherEndian<DefaultOptions, BigEndian>;
+type ChangedState = HashMap<H256, (Maybe<AccountState>, HashMap<H256, H256>)>;
 lazy_static! {
     static ref CODER: BincodeOpts = DefaultOptions::new().with_big_endian();
 }
@@ -63,6 +65,7 @@ pub enum Error {
 
 const BACKUP_SUBDIR: &str = "backup";
 const CUSTOM_LOCATION: &str = "tmp_inner_space";
+const NUM_ENTRIES_IN_STORAGES_CHUNK: usize = 10000;
 
 /// Marker-like wrapper for cleaning temporary directory.
 /// Temporary directory is only used in tests.
@@ -192,7 +195,6 @@ impl Descriptors {
     }
 }
 
-
 impl<D> Storage<D>
 where
     D: DBInner,
@@ -255,7 +257,6 @@ type ReadOnlyDb = rocksdb::DBWithThreadMode<rocksdb::SingleThreaded>;
 pub type StorageSecondary = Storage<DBWithThreadModeInner>;
 
 impl StorageSecondary {
-
     pub fn open_secondary_persistent<P: AsRef<Path>>(path: P, gc_enabled: bool) -> Result<Self> {
         Self::open(Location::Persisent(path.as_ref().to_owned()), gc_enabled)
     }
@@ -303,9 +304,11 @@ impl Storage<OptimisticTransactionDBInner> {
     pub fn create_temporary() -> Result<Self> {
         Self::open(Location::Temporary(Arc::new(TempDir::new()?)), false)
     }
+
     pub fn create_temporary_gc() -> Result<Self> {
         Self::open(Location::Temporary(Arc::new(TempDir::new()?)), true)
     }
+
     fn open(location: Location, gc_enabled: bool) -> Result<Self> {
         log::warn!("gc_enabled {}", gc_enabled);
         log::info!("location is {:?}", location);
@@ -379,6 +382,7 @@ impl Storage<OptimisticTransactionDBInner> {
         FixedSecureTrieMut::new(DatabaseTrieMut::trie_for(handle, root))
     }
 
+    // FIXME: flush_changes_hashed code duplication
     pub fn flush_changes(&self, state_root: H256, state_updates: crate::ChangedState) -> H256 {
         let r = self.rocksdb_trie_handle();
 
@@ -437,6 +441,65 @@ impl Storage<OptimisticTransactionDBInner> {
             .leak_root()
     }
 
+    // FIXME: flush_changes code duplication
+    pub fn flush_changes_hashed(&self, state_root: H256, state_updates: ChangedState) -> H256 {
+        let r = self.rocksdb_trie_handle();
+
+        let db_trie = TrieCollection::new(r);
+
+        use triedb::TrieMut;
+        let mut accounts = db_trie.trie_for(state_root);
+
+        for (address, (state, storages)) in state_updates {
+            if let Maybe::Just(AccountState {
+                nonce,
+                balance,
+                code,
+            }) = state
+            {
+                let mut account: Account = accounts
+                    .get(address.as_bytes())
+                    .and_then(|accounts| rlp::decode(&accounts).ok())
+                    .unwrap_or_default();
+
+                account.nonce = nonce;
+                account.balance = balance;
+
+                if !code.is_empty() {
+                    let code_hash = code.hash();
+                    self.set::<Codes>(code_hash, code);
+                    account.code_hash = code_hash;
+                }
+
+                let storage_values = storages.into_iter().chunks(NUM_ENTRIES_IN_STORAGES_CHUNK);
+                for index_changes in storage_values.into_iter() {
+                    let mut storage = db_trie.trie_for(account.storage_root);
+                    for (index, value) in index_changes {
+                        if value != H256::default() {
+                            let value = U256::from_big_endian(&value[..]);
+                            storage.insert(index.as_bytes(), &rlp::encode(&value));
+                        } else {
+                            storage.delete(index.as_bytes());
+                        }
+                    }
+
+                    let storage_patch = storage.into_patch();
+                    account.storage_root = db_trie
+                        .apply_increase(storage_patch, |_| vec![])
+                        .leak_root()
+                }
+                accounts.insert(address.as_bytes(), &rlp::encode(&account));
+            } else {
+                accounts.delete(address.as_bytes());
+            }
+        }
+
+        let accounts_patch = accounts.into_patch();
+        db_trie
+            .apply_increase(accounts_patch, account_extractor)
+            .leak_root()
+    }
+
     pub fn merge_from_db(&self, other_db: &Self) -> Result<()> {
         assert!(!self.gc_enabled, "Cannot merge to db with rc counters");
         assert!(
@@ -610,8 +673,7 @@ impl Storage<OptimisticTransactionDBInner> {
         if !self.gc_enabled {
             return (vec![], vec![]);
         }
-        self
-            .rocksdb_trie_handle()
+        self.rocksdb_trie_handle()
             .gc_cleanup_layer(removes, account_extractor)
     }
 
@@ -627,6 +689,65 @@ impl Storage<OptimisticTransactionDBInner> {
         engine.create_new_backup_flush(self.db.as_ref(), true)?;
         Ok(backup_dir)
     }
+
+    pub fn set_initial(
+        &mut self,
+        accounts: impl IntoIterator<Item = (H256, evm::backend::MemoryAccount)>,
+        state_root: H256,
+    ) -> H256 {
+        use std::collections::hash_map::Entry::*;
+
+        fn set_account_state(
+            state_updates: &mut ChangedState,
+            address: H256,
+            account_state: AccountState,
+        ) {
+            match state_updates.entry(address) {
+                Occupied(mut e) => {
+                    e.get_mut().0 = Maybe::Just(account_state);
+                }
+                Vacant(e) => {
+                    e.insert((Maybe::Just(account_state), HashMap::new()));
+                }
+            };
+        }
+
+        fn ext_storage(
+            state_updates: &mut ChangedState,
+            address: H256,
+            indexed_values: impl IntoIterator<Item = (H256, H256)>,
+        ) {
+            let (_, storage) = state_updates
+                .entry(address)
+                .or_insert_with(|| (Maybe::Just(AccountState::default()), HashMap::new()));
+
+            storage.extend(indexed_values);
+        }
+
+        let mut state_updates: ChangedState = HashMap::new();
+
+        for (
+            address,
+            evm::backend::MemoryAccount {
+                nonce,
+                balance,
+                storage,
+                code,
+            },
+        ) in accounts
+        {
+            let account_state = AccountState {
+                nonce,
+                balance,
+                code: code.into(),
+            };
+
+            set_account_state(&mut state_updates, address, account_state);
+            ext_storage(&mut state_updates, address, storage);
+        }
+
+        self.flush_changes_hashed(state_root, state_updates)
+    }
 }
 
 static SECONDARY_MODE_PATH_SUFFIX: &str = "velas-secondary";

genesis-utils/src/lib.rs

@@ -2,7 +2,7 @@ use {
     solana_download_utils::download_genesis_if_missing,
     solana_runtime::hardened_unpack::unpack_genesis_archive,
     solana_sdk::{
-        genesis_config::{GenesisConfig, DEFAULT_GENESIS_ARCHIVE},
+        genesis_config::{GenesisConfig, DEFAULT_GENESIS_ARCHIVE, EVM_GENESIS_ARCHIVE},
         hash::Hash,
     },
     std::net::SocketAddr,
@@ -50,22 +50,36 @@ pub fn download_then_check_genesis_hash(
         return Ok(genesis_config);
     }
 
-    let genesis_package = ledger_path.join(DEFAULT_GENESIS_ARCHIVE);
-    let genesis_config = if let Ok(tmp_genesis_package) =
-        download_genesis_if_missing(rpc_addr, &genesis_package, use_progress_bar)
+    let genesis_package_native = ledger_path.join(DEFAULT_GENESIS_ARCHIVE);
+    let genesis_package_evm = ledger_path.join(EVM_GENESIS_ARCHIVE);
+
+    let genesis_config = if let Ok((tmp_genesis_native, tmp_genesis_evm)) =
+        download_genesis_if_missing(rpc_addr, &ledger_path, use_progress_bar)
     {
         unpack_genesis_archive(
-            &tmp_genesis_package,
+            &tmp_genesis_native,
             ledger_path,
             max_genesis_archive_unpacked_size,
         )
-        .map_err(|err| format!("Failed to unpack downloaded genesis config: {}", err))?;
+        .map_err(|err| format!("Failed to unpack downloaded native genesis config: {}", err))?;
+
+        unpack_genesis_archive(
+            &tmp_genesis_evm,
+            ledger_path,
+            max_genesis_archive_unpacked_size,
+        )
+        .map_err(|err| format!("Failed to unpack downloaded evm genesis config: {}", err))?;
 
         let downloaded_genesis = GenesisConfig::load(ledger_path)
             .map_err(|err| format!("Failed to load downloaded genesis config: {}", err))?;
 
         check_genesis_hash(&downloaded_genesis, expected_genesis_hash)?;
-        std::fs::rename(tmp_genesis_package, genesis_package)
+        // NOTE: evm genesis hash/state_root check?
+
+        std::fs::rename(tmp_genesis_native, genesis_package_native)
+            .map_err(|err| format!("Unable to rename: {:?}", err))?;
+
+        std::fs::rename(tmp_genesis_evm, genesis_package_evm)
             .map_err(|err| format!("Unable to rename: {:?}", err))?;
 
         downloaded_genesis