diff --git a/Cargo.lock b/Cargo.lock index 50ce03c386..addad67490 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5511,10 +5511,12 @@ dependencies = [ "assert_matches", "bitvec", "blockifier", + "dotenv", "lazy_static", "mc-analytics", "mc-block-import", "mc-db", + "mc-devnet", "mc-exec", "mc-mempool", "mockall", @@ -5535,10 +5537,14 @@ dependencies = [ "opentelemetry_sdk", "proptest", "proptest-derive", + "rand", "rstest 0.18.2", + "serde", "serde_json", + "starknet", "starknet-core", "starknet-types-core 0.1.7 (git+https://github.com/kasarlabs/types-rs.git?branch=feat-deserialize-v0.1.7)", + "starknet-types-rpc", "starknet_api", "thiserror 2.0.3", "tokio", @@ -5575,6 +5581,7 @@ dependencies = [ "rayon", "rocksdb", "serde", + "serde_json", "starknet-types-core 0.1.7 (git+https://github.com/kasarlabs/types-rs.git?branch=feat-deserialize-v0.1.7)", "starknet-types-rpc", "starknet_api", diff --git a/Dockerfile b/Dockerfile index 4d163307ab..5bf191cd07 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN apt-get -y update && \ # Set the working directory WORKDIR /usr/src/madara/ # Copy the source code into the container -COPY Cargo.toml Cargo.lock ./ +COPY Cargo.toml Cargo.lock .db-versions.yml ./ COPY crates crates COPY .db-versions.yml .db-versions.yml ./ COPY cairo-artifacts cairo-artifacts diff --git a/configs/presets/devnet.yaml b/configs/presets/devnet.yaml index 4e562cbc9d..b820e4182f 100644 --- a/configs/presets/devnet.yaml +++ b/configs/presets/devnet.yaml @@ -5,8 +5,8 @@ gateway_url: "http://localhost:8080/gateway/" native_fee_token_address: "0x04718f5a0fc34cc1af16a1cdee98ffb20c31f5cd61d6ab07201858f4287c938d" parent_fee_token_address: "0x049d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7" latest_protocol_version: "0.13.2" -block_time: "10s" -pending_block_update_time: "2s" +block_time: "1h" +pending_block_update_time: "1s" execution_batch_size: 16 bouncer_config: block_max_capacity: diff --git a/crates/madara/client/analytics/src/lib.rs b/crates/madara/client/analytics/src/lib.rs index b75a90b68b..ce20705d89 100644 --- a/crates/madara/client/analytics/src/lib.rs +++ b/crates/madara/client/analytics/src/lib.rs @@ -51,7 +51,9 @@ impl Analytics { self.meter_provider = Some(self.init_metric_provider()?); let layer = OpenTelemetryTracingBridge::new(&logger_provider); - tracing_subscriber.with(OpenTelemetryLayer::new(tracer)).with(layer).init(); + tracing_subscriber.with(OpenTelemetryLayer::new(tracer)).init(); + + tracing::info!("OTEL initialized"); Ok(()) } diff --git a/crates/madara/client/block_production/Cargo.toml b/crates/madara/client/block_production/Cargo.toml index ca4311a1a2..7214377e53 100644 --- a/crates/madara/client/block_production/Cargo.toml +++ b/crates/madara/client/block_production/Cargo.toml @@ -26,6 +26,7 @@ blockifier = { workspace = true, features = ["testing"] } mockall.workspace = true assert_matches.workspace = true lazy_static.workspace = true +serde = { workspace = true, default-features = true } serde_json.workspace = true [features] @@ -37,6 +38,7 @@ testing = ["blockifier/testing", "mc-db/testing", "mockall"] mc-analytics.workspace = true mc-block-import.workspace = true mc-db.workspace = true +mc-devnet.workspace = true mc-exec.workspace = true mc-mempool.workspace = true mp-block.workspace = true @@ -53,7 +55,8 @@ blockifier.workspace = true starknet-core.workspace = true starknet-types-core.workspace = true starknet_api.workspace = true - +starknet.workspace = true +starknet-types-rpc = { workspace = true } # Other anyhow.workspace = true mockall = { workspace = true, optional = true } @@ -76,3 +79,5 @@ tracing = { workspace = true } tracing-core = { workspace = true, default-features = false } tracing-opentelemetry = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter"] } +rand.workspace = true +dotenv.workspace = true diff --git a/crates/madara/client/block_production/src/lib.rs b/crates/madara/client/block_production/src/lib.rs index b5783b8dd3..afb63c67f0 100644 --- a/crates/madara/client/block_production/src/lib.rs +++ b/crates/madara/client/block_production/src/lib.rs @@ -17,31 +17,41 @@ use crate::close_block::close_block; use crate::metrics::BlockProductionMetrics; -use blockifier::blockifier::transaction_executor::{TransactionExecutor, BLOCK_STATE_ACCESS_ERR}; +use blockifier::blockifier::transaction_executor::{ + TransactionExecutor, TransactionExecutorError, BLOCK_STATE_ACCESS_ERR, +}; use blockifier::bouncer::BouncerWeights; use blockifier::transaction::errors::TransactionExecutionError; +use blockifier::transaction::objects::TransactionExecutionInfo; +use dotenv::dotenv; use finalize_execution_state::StateDiffToStateMapError; use mc_block_import::{BlockImportError, BlockImporter}; use mc_db::db_block_id::DbBlockId; use mc_db::{MadaraBackend, MadaraStorageError}; +use mc_devnet::{Call, Multicall, Selector}; use mc_exec::{BlockifierStateAdapter, ExecutionContext}; use mc_mempool::header::make_pending_header; -use mc_mempool::{L1DataProvider, MempoolProvider}; +use mc_mempool::{transaction_hash, L1DataProvider, MempoolProvider}; use mp_block::{BlockId, BlockTag, MadaraPendingBlock, VisitedSegments}; use mp_class::compile::ClassCompilationError; use mp_class::ConvertedClass; use mp_convert::ToFelt; use mp_receipt::from_blockifier_execution_info; use mp_state_update::{ContractStorageDiffItem, DeclaredClassItem, NonceUpdate, StateDiff, StorageEntry}; -use mp_transactions::TransactionWithHash; +use mp_transactions::{BroadcastedTransactionExt as _, TransactionWithHash}; use mp_utils::service::ServiceContext; use opentelemetry::KeyValue; +use rand::{thread_rng, Rng}; +use starknet::signers::SigningKey; +use starknet_api::transaction::EventKey; use starknet_types_core::felt::Felt; +use starknet_types_rpc::{BroadcastedInvokeTxn, BroadcastedTxn, InvokeTxnV1}; use std::borrow::Cow; use std::collections::VecDeque; -use std::mem; +use std::str::FromStr as _; use std::sync::Arc; use std::time::Instant; +use std::{env, mem}; mod close_block; mod finalize_execution_state; @@ -59,6 +69,15 @@ struct ContinueBlockStats { pub n_rejected: usize, } +// TODO: might wanna remove the 0 in the start +const SPAWNED_BOT_EVENT_SELECTOR: &str = "0x2cd0383e81a65036ae8acc94ac89e891d1385ce01ae6cc127c27615f5420fa3"; +const BOMB_FOUND_EVENT_SELECTOR: &str = "0x111861367b42e77c11a98efb6d09a14c2dc470eee1a4d2c3c1e8c54015da2e5"; +const DIAMOND_FOUND_EVENT_SELECTOR: &str = "0x14528085c8fd64b9210572c5b6015468f8352c17c9c22f5b7aa62a55a56d8d7"; +const TILE_MINED_SELECTOR: &str = "0xd5efc9cfb6a4f6bb9eae0ce39d32480473877bb3f7a4eaa3944c881a2c8d25"; +const TILE_ALREADY_MINED_SELECTOR: &str = "0x1b74d97806c93468070e49a1626aba00f8e89dfb07246492af4566f898de982"; +const SUSPEND_BOT_SELECTOR: &str = "0x1dcca826eea45d96bfbf26e9aabf510e94c6de62d0ce5e5b6e60c51c7640af8"; +const REVIVE_BOT_SELECTOR: &str = "0x1d6a6a42fd13b206a721dbca3ae720621707ef3016850e2c5536244e5a7858a"; + #[derive(Debug, thiserror::Error)] pub enum Error { #[error("Storage error: {0:#}")] @@ -291,6 +310,19 @@ impl BlockProductionTask { // Execute the transactions. let all_results = self.executor.execute_txs(&txs_to_process_blockifier); + + // println!(">>> Execution returned with : {:?} within {:?} ", all_results.len(), end); + + // println!(">>> TXNS TO PROCESS BLOCKFIER LENGTH : {:?} ", txs_to_process_blockifier.len()); + // println!(">>> TXNS TO PROCESS BLOCKFIER : {:?} ", txs_to_process_blockifier); + + // let result: &Vec> = &all_results.as_ref(); + // let x = result.iter().map(|x| x.as_ref().unwrap()).collect::>()[0]; + // let n_steps = x.transaction_receipt.resources.vm_resources.n_steps; + // println!(">>> N_STEPS {:?}", n_steps); + + let _ress = self.listen_for_bot_events(&all_results).expect("Couldn't ingest Bot events"); + // When the bouncer cap is reached, blockifier will return fewer results than what we asked for. block_now_full = all_results.len() < txs_to_process_blockifier.len(); @@ -317,6 +349,8 @@ impl BlockProductionTask { self.declared_classes.push(class); } + // TODO: add here the event listening logic + self.block .inner .receipts @@ -367,6 +401,166 @@ impl BlockProductionTask { Ok(ContinueBlockResult { state_diff, visited_segments, bouncer_weights, stats, block_now_full }) } + pub fn listen_for_bot_events( + &mut self, + all_txns: &Vec< + Result< + blockifier::transaction::objects::TransactionExecutionInfo, + blockifier::blockifier::transaction_executor::TransactionExecutorError, + >, + >, + ) -> Result<(), Error> { + // search through all_txns and get SpawnedBot or BombFound events + + let mut spawned_bots: Vec = Vec::new(); + let mut killed_bots: Vec = Vec::new(); + + // events: [OrderedEvent { order: 0, event: EventContent { keys: [EventKey(0x2cd0383e81a65036ae8acc94ac89e891d1385ce01ae6cc127c27615f5420fa3)], + // data: EventData([0x7484e8e3af210b2ead47fa08c96f8d18b616169b350a8b75fe0dc4d2e01d493, 0x1c9, 0x66dbd884899534c3ba7216743e8d0a683e3c5b5b8cac37441f55c1b43a8019c]) } }], + + // The logic below is written assuming that the event will have one key and 3 values. + + for tx in all_txns { + if tx.is_err() { + continue; + } + let tx = tx.as_ref().unwrap(); + + let execute_call_info = &tx.execute_call_info.as_ref(); + + if let Some(execute_call) = execute_call_info { + let inner_calls: &Vec = execute_call.inner_calls.as_ref(); + + for values in inner_calls { + let ordered_events: &Vec = + values.execution.events.as_ref(); + + for ordered_event in ordered_events.iter() { + let event = ordered_event.event.to_owned(); + + let spawned_bot_felt = Felt::from_str(SPAWNED_BOT_EVENT_SELECTOR) + .expect("Unable to convert selector string to felt"); // Or however you get your Felt value + + let bomb_found_felt = Felt::from_str(BOMB_FOUND_EVENT_SELECTOR) + .expect("Unable to convert selector string to felt"); // Or however you get your Felt value + + let diamond_found_felt = Felt::from_str(DIAMOND_FOUND_EVENT_SELECTOR) + .expect("Unable to convert selector string to felt"); // Or however you get your Felt value + + let tile_mined_felt = + Felt::from_str(TILE_MINED_SELECTOR).expect("Unable to convert selector string to felt"); // Or however you get your Felt value + + let tile_already_mined_felt = Felt::from_str(TILE_ALREADY_MINED_SELECTOR) + .expect("Unable to convert selector string to felt"); // Or however you get your Felt value + + let suspend_bot_felt = + Felt::from_str(SUSPEND_BOT_SELECTOR).expect("Unable to convert selector string to felt"); // Or however you get your Felt value + + let revive_bot_felt = + Felt::from_str(REVIVE_BOT_SELECTOR).expect("Unable to convert selector string to felt"); // Or however you get your Felt value + + for key in event.keys { + // BombFound + if key == EventKey(bomb_found_felt) { + let bot_address = event.data.0[0].to_string(); + let bot_location = event.data.0[1].to_string(); + println!( + ">>> Event : BombFound by {:?} at {:?}", + Felt::from_str(bot_address.as_str()).expect("Could not get address"), + bot_location + ); + killed_bots.push(bot_address); + } + // DiamondFound + else if key == EventKey(diamond_found_felt) { + let bot_address = event.data.0[0].to_string(); + let bot_points = event.data.0[1].to_string(); + let bot_location = event.data.0[2].to_string(); + println!( + ">>> Event : DiamondFound by {:?} at {:?} for {:?}", + Felt::from_str(bot_address.as_str()).expect("Could not get address"), + bot_location, + bot_points + ); + } + // TileMined + else if key == EventKey(tile_mined_felt) { + let bot_address = event.data.0[0].to_string(); + let points = event.data.0[1].to_string(); + let location = event.data.0[2].to_string(); + println!( + ">>> Event : TileMined by {:?} at {:?} for {:?}", + Felt::from_str(bot_address.as_str()).expect("Could not get address"), + location, + points + ); + self.backend + .game_update_metadata(|meta| { + meta.tiles_mined += 1; + }) + .expect("could not update the tiles mined number"); + } + // TileAlreadyMined + else if key == EventKey(tile_already_mined_felt) { + let bot_address = event.data.0[0].to_string(); + let bot_location = event.data.0[1].to_string(); + println!( + ">>> Event : TileAlreadyMined {:?} at {:?}", + Felt::from_str(bot_address.as_str()).expect("Could not get address"), + bot_location + ); + } + // SpawnedBot + else if key == EventKey(spawned_bot_felt) { + let bot_address = event.data.0[0].to_string(); + let player = event.data.0[1].to_string(); + let bot_location = event.data.0[2].to_string(); + println!( + ">>> Event : SpawnedBot {:?} at {:?} by {:?}", + Felt::from_str(bot_address.as_str()).expect("Could not get address"), + bot_location, + Felt::from_str(player.as_str()).expect("Could not get address") + ); + spawned_bots.push(bot_address); + } + // SuspendBot + else if key == EventKey(suspend_bot_felt) { + let bot_address = event.data.0[0].to_string(); + println!( + ">>> Event : SuspendBot {:?}", + Felt::from_str(bot_address.as_str()).expect("Could not get address") + ); + // TODO: add kill bot here if needed. + } + // ReviveBot + else if key == EventKey(revive_bot_felt) { + let bot_address = event.data.0[0].to_string(); + println!( + ">>> Event : ReviveBot {:?}", + Felt::from_str(bot_address.as_str()).expect("Could not get address") + ); + // TODO: add spawned bot here if needed. + } + } + } + } + } + } + + // Kill all the bots ! + // TODO: these are multiple DB operations, can it be clubbed to a single operation + for killed_bot in killed_bots { + let _ = self.backend.game_delete_bot_address(&killed_bot.as_str()).expect("Could not remove the bot"); + } + + // Add new bots ! + for spawned_bot in spawned_bots { + let _ = self.backend.game_add_bot_address(&spawned_bot.as_str()).expect("Could not add the bot"); + } + + Ok(()) + } + /// Closes the current block and prepares for the next one #[tracing::instrument(skip(self), fields(module = "BlockProductionTask"))] async fn close_and_prepare_next_block( @@ -481,6 +675,8 @@ impl BlockProductionTask { #[tracing::instrument(skip(self), fields(module = "BlockProductionTask"))] pub async fn on_pending_time_tick(&mut self) -> Result { + dotenv().ok(); + let start = Instant::now(); let current_pending_tick = self.current_pending_tick; if current_pending_tick == 0 { return Ok(false); @@ -527,6 +723,44 @@ impl BlockProductionTask { // do not forget to flush :) self.backend.flush().map_err(|err| BlockImportError::Internal(format!("DB flushing error: {err:#}").into()))?; + // TODO: Measure the transactions time -------------------------------------------------- + // TODO: Do all of it inside a function + // ========================================================================================= + // Execute BOT transactions : + + let game_width = env::var("MADARA_GAME_WIDTH").expect("MADARA_GAME_WIDTH not set").parse::().unwrap(); + let game_height = env::var("MADARA_GAME_HEIGHT").expect("MADARA_GAME_HEIGHT not set").parse::().unwrap(); + + println!(">>> Game width : {:?} and height : {:?}", game_width, game_height); + + let area = game_width * game_height; + + let game_metadata = self.backend.game_get_metadata().expect("Unable to fetch last start index"); + let bot_addresses = self.backend.game_get_bots_list().expect("Could not get bots' list"); + if game_metadata.tiles_mined < area && bot_addresses.len() > 0 { + println!(">>> Triggering bot transactions"); + println!(">>> Current Tiles mined : {:?} vs total to be mined : {:?}", game_metadata.tiles_mined, area); + let addresses_clone = bot_addresses.clone(); + + // let c = bot_addresses + // .iter() + // .map(|x| Felt::from_str(x).expect("could not convert string to felt")) + // .collect::>(); + + // println!(">>> DB bots list : {:?}", c); + // Do nothing if 0 bots to execute + if bot_addresses.is_empty() { + return Ok(false); + } + // TODO: check if game is active or not + // TODO: what is bot is disabled ? + + let txn = self.generate_txns(addresses_clone); + self.mempool.tx_accept_invoke(txn).expect("Unable to accept invoke tx"); + println!(">>> Time taken to run on_pending_tick: {:?}", start.elapsed().as_millis()); + + // ========================================================================================= + } Ok(false) } @@ -617,6 +851,74 @@ impl BlockProductionTask { Ok(()) } + fn generate_txns(&self, contract_addresses: Vec) -> BroadcastedInvokeTxn { + let x = env::var("MADARA_GAME_SEQUENCER_ADDRESS").unwrap(); + let y = env::var("MADARA_GAME_SEQUENCER_PRIVATE_KEY").unwrap(); + let z = env::var("MADARA_GAME_CONTRACT_ADDRESS").unwrap(); + + println!(">>> SEQUENCER ADDRESS {:?}", x); + println!(">>> SEQUENCER PRIVATE KEY {:?}", y); + println!(">>> GAME ADDRESS {:?}", z); + + let sequencer_address = Felt::from_hex(&x.as_str()).expect("Unable to extract public key from hex"); + let sequencer_priv_key = Felt::from_hex(&y.as_str()).expect("Unable to extract priv key from hex"); + let game_address = Felt::from_hex(&z.as_str()).expect("Unable to extract public key from hex"); + + let signing_key = SigningKey::from_secret_scalar(sequencer_priv_key); + + let nonce = self + .backend + .get_contract_nonce_at(&DbBlockId::Pending, &sequencer_address) + .expect("Unable to fetch nonce from the block.") + // if nonce is not found, use 0 + .unwrap_or(Felt::from(0)); + + println!(">>> TXN NONCE {:?}", nonce); + + let mut call_vec = Vec::new(); + for address in contract_addresses { + let random_seed: u64 = thread_rng().gen(); + call_vec.push(Call { + to: game_address, + selector: Selector::from("mine"), + calldata: vec![Felt::from_str(&address).unwrap(), Felt::from(random_seed)], + }) + } + + let txn = BroadcastedTxn::Invoke(BroadcastedInvokeTxn::V1(InvokeTxnV1 { + sender_address: sequencer_address, + calldata: Multicall::with_vec(call_vec).flatten().collect(), + max_fee: Felt::from_str("100000000").unwrap(), + signature: vec![], // will be added when signing + nonce, + })); + + let signed_transaction = self.sign_tx(txn, signing_key.clone()).expect("Not able to sign the transaction."); + + match signed_transaction { + BroadcastedTxn::Invoke(tx) => tx, + _ => panic!("Invalid Txn"), + } + } + + fn sign_tx(&self, mut tx: BroadcastedTxn, signing_key: SigningKey) -> anyhow::Result> { + let (blockifier_tx, _) = BroadcastedTxn::into_blockifier( + tx.clone(), + self.backend.chain_config().chain_id.to_felt(), + self.backend.chain_config().latest_protocol_version, + )?; + let signature = signing_key.sign(&transaction_hash(&blockifier_tx))?; + let tx_signature = match &mut tx { + BroadcastedTxn::Invoke(tx) => match tx { + BroadcastedInvokeTxn::V1(tx) => &mut tx.signature, + _ => panic!("Invalid Txn"), + }, + _ => panic!("Invalid Txn"), + }; + *tx_signature = vec![signature.r, signature.s]; + Ok(tx) + } + fn block_n(&self) -> u64 { self.executor.block_context.block_info().block_number.0 } @@ -624,6 +926,128 @@ impl BlockProductionTask { #[cfg(test)] mod tests { + use std::str::FromStr; + + use super::*; + use starknet_types_core::felt::Felt; + + // Helper function to create test bots + fn create_test_bots(count: i64) -> Vec { + // format : 0x + (0..count).map(|i| Felt::from_str(&format!("0x{i}")).unwrap()).collect() + } + + // Helper function to verify distribution properties + fn verify_distribution(result: &Vec, expected_len: i64, original_bots: &Vec) { + assert_eq!(result.len(), expected_len as usize, "Result length should match MINES_PER_TRANSACTION"); + + for bot in result { + assert!(original_bots.contains(bot), "Every bot in result should exist in original list"); + } + } + + // These tests can be more stressing + // #[test] + // fn test_less_bots_than_mines() { + // let num_bots: i64 = 137; + // let bots = create_test_bots(num_bots); // 100 bots < 500 MINES_PER_TRANSACTION + // let start_index = 124; + + // // Array : 51 to 100, then 1 to 100 4 times, then 1 to 50 + + // let (result, new_start_index) = get_bots_list(); + + // verify_distribution(&result, MINES_PER_TRANSACTION, &bots); + + // // Each bot should appear multiple times + // let repetitions = MINES_PER_TRANSACTION / num_bots; + // let mut count_map = std::collections::HashMap::new(); + // for bot in &result { + // *count_map.entry(bot).or_insert(0) += 1; + // } + + // println!("test_less_bots_than_mines : {:?}", result); + // println!("test_less_bots_than_mines last index : {:?}", new_start_index); + + // // Check if each bot appears at least the minimum number of times + // for count in count_map.values() { + // assert!(*count >= repetitions, "Each bot should appear at least {} times", repetitions); + // } + // } + + // #[test] + // fn test_equal_bots_to_mines() { + // let bots = create_test_bots(MINES_PER_TRANSACTION); + // let start_index = 36; + + // let (result, new_start_index) = get_bots_list(bots.clone(), start_index); + + // verify_distribution(&result, MINES_PER_TRANSACTION, &bots); + + // // Each bot should appear exactly once + // let mut count_map = std::collections::HashMap::new(); + // for bot in &result { + // *count_map.entry(bot).or_insert(0) += 1; + // } + + // println!("test_equal_bots_to_mines : {:?}", result); + // println!("test_equal_bots_to_mines last index : {:?}", new_start_index); + + // for count in count_map.values() { + // assert_eq!(*count, 1, "Each bot should appear exactly once"); + // } + // } + + // #[test] + // fn test_more_bots_than_mines() { + // let bots = create_test_bots(3110); // 1000 bots > 500 MINES_PER_TRANSACTION + // let start_index = 457; + + // let (result, new_start_index) = get_bots_list(bots.clone(), start_index); + + // verify_distribution(&result, MINES_PER_TRANSACTION, &bots); + + // // Each bot should appear at most once + // let mut count_map = std::collections::HashMap::new(); + // for bot in &result { + // *count_map.entry(bot).or_insert(0) += 1; + // } + + // println!("test_more_bots_than_mines : {:?}", result); + // println!("test_more_bots_than_mines last index : {:?}", new_start_index); + + // for count in count_map.values() { + // assert_eq!(*count, 1, "Each bot should appear exactly once"); + // } + + // // Verify that we start from last_index + // assert_eq!(result[0], bots[start_index as usize], "First bot should match the last_index position"); + // } + + // #[test] + // fn test_empty_bots_list() { + // let bots = Vec::new(); + // let last_index = 0; + + // let (result, new_index) = get_bots_list(bots, last_index); + + // assert_eq!(result.len(), 0, "Result should be empty for empty input"); + // assert_eq!(new_index, 0, "New index should be 0 for empty input"); + // } + + // #[test] + // fn test_index_wrapping() { + // let bots = create_test_bots(200); + // let last_index = 150; // Near the end of the list + + // let (result, new_index) = get_bots_list(bots.clone(), last_index); + + // verify_distribution(&result, MINES_PER_TRANSACTION, &bots); + + // // Verify that the list wraps around correctly + // assert!(new_index < bots.len() as i64, "New index should wrap around"); + // } + use std::{collections::HashMap, sync::Arc}; use blockifier::{ @@ -643,7 +1067,6 @@ mod tests { core::{ClassHash, ContractAddress, PatriciaKey}, felt, patricia_key, }; - use starknet_types_core::felt::Felt; use crate::{ finalize_execution_state::state_map_to_state_diff, metrics::BlockProductionMetrics, BlockProductionTask, diff --git a/crates/madara/client/db/Cargo.toml b/crates/madara/client/db/Cargo.toml index 0f9c30e2d1..33863bfca2 100644 --- a/crates/madara/client/db/Cargo.toml +++ b/crates/madara/client/db/Cargo.toml @@ -42,6 +42,7 @@ librocksdb-sys = { workspace = true } rayon = { workspace = true } rocksdb.workspace = true serde = { workspace = true } +serde_json = { workspace = true } tempfile = { workspace = true, optional = true } thiserror = { workspace = true } tokio = { workspace = true, features = [ diff --git a/crates/madara/client/db/src/game_db.rs b/crates/madara/client/db/src/game_db.rs new file mode 100644 index 0000000000..4df1c1b762 --- /dev/null +++ b/crates/madara/client/db/src/game_db.rs @@ -0,0 +1,234 @@ +use crate::DatabaseExt; +use crate::{Column, MadaraBackend}; +use rocksdb::WriteBatch; +use serde::{Deserialize, Serialize}; +use std::env; + +#[derive(Serialize, Deserialize)] +pub struct AddressNode { + address: String, + next: Option, // Next address in sequence + previous: Option, // Previous address in sequence +} + +#[derive(Serialize, Deserialize, Debug, Default)] +pub struct ListMetadata { + game_address: String, + head: Option, // First address + tail: Option, // Last address + pub next_iter_addr: Option, // Next address for iterator + pub length: u64, // Number of addresses in list + pub tiles_mined: u64, // Number of tiles mined +} + +const METADATA_KEY: &[u8] = b"list_metadata"; + +impl MadaraBackend { + pub fn game_get_metadata(&self) -> Result { + let col = self.db.get_column(Column::Game); + let meta = self + .db + .get_cf(&col, METADATA_KEY)? + .map(|bytes| serde_json::from_slice(&bytes).unwrap()) + .unwrap_or(ListMetadata::default()); + Ok(meta) + } + + pub fn game_update_metadata(&self, updates: impl FnOnce(&mut ListMetadata)) -> Result<(), rocksdb::Error> { + let col = self.db.get_column(Column::Game); + let mut batch = WriteBatch::default(); + + // Get current metadata or create default if none exists + let mut metadata = self + .db + .get_cf(&col, METADATA_KEY)? + .map(|bytes| serde_json::from_slice(&bytes).unwrap()) + .unwrap_or(ListMetadata::default()); + + // Apply the updates to the metadata + updates(&mut metadata); + + // Serialize and write the updated metadata + let serialized = serde_json::to_vec(&metadata).unwrap(); + batch.put_cf(&col, METADATA_KEY, serialized); + + // Write the batch to the database + self.db.write(batch)?; + + Ok(()) + } + + /// Add new bot address to the linked list + /// 1. Create new node with current address, with previous pointing to old tail + /// 2. Add new node to the database + /// 3. If it's first address, set head & tail to this address + /// 4. Update old tail's next field to point to new address + /// 5. Update tail to the new address + /// 6. Update metadata, including tail and length + /// 7. Write batch to the database + pub fn game_add_bot_address(&self, address: &str) -> Result<(), rocksdb::Error> { + let col = self.db.get_column(Column::Game); + let mut meta = self.game_get_metadata()?; + + let new_node = AddressNode { + address: address.to_string(), + next: None, + previous: meta.tail.clone(), // Point to old tail + }; + + let mut batch = WriteBatch::default(); + + // If list is empty + if meta.head.is_none() { + meta.head = Some(address.to_string()); + meta.tail = Some(address.to_string()); + } else { + // Update old tail to point to new address + if let Some(old_tail) = &meta.tail { + let mut old_tail_node: AddressNode = + serde_json::from_slice(&self.db.get_cf(&col, old_tail.as_bytes())?.unwrap()).unwrap(); + old_tail_node.next = Some(address.to_string()); + batch.put_cf(&col, old_tail.as_bytes(), serde_json::to_vec(&old_tail_node).unwrap()); + } + meta.tail = Some(address.to_string()); + } + + meta.length += 1; + + batch.put_cf(&col, address.as_bytes(), serde_json::to_vec(&new_node).unwrap()); + batch.put_cf(&col, METADATA_KEY, serde_json::to_vec(&meta).unwrap()); + + // Added New Node + // Updated old tail to point to new node + // Incremented metadata length by 1 + + self.db.write(batch) + } + + /// Delete a game address from the list + /// 1. Modify next pointer of previous node to point to node after this one + /// 2. Modify previous pointer of next node to point to node before this one + /// 3. Delete provided node. Reduce length by 1 in metadata + /// 4. Write batch to the database + pub fn game_delete_bot_address(&self, address: &str) -> Result<(), rocksdb::Error> { + let col = self.db.get_column(Column::Game); + let mut meta = self.game_get_metadata()?; + + // Get the node to delete + if let Some(node_bytes) = self.db.get_cf(&col, address.as_bytes())? { + let node: AddressNode = serde_json::from_slice(&node_bytes).unwrap(); + let mut batch = WriteBatch::default(); + + // Update previous node's next pointer + if let Some(prev_addr) = &node.previous { + let mut prev_node: AddressNode = + serde_json::from_slice(&self.db.get_cf(&col, prev_addr.as_bytes())?.unwrap()).unwrap(); + prev_node.next = node.next.clone(); + batch.put_cf(&col, prev_addr.as_bytes(), serde_json::to_vec(&prev_node).unwrap()); + } else { + // Deleting head + meta.head = node.next.clone(); + } + + // Update next node's previous pointer + if let Some(next_addr) = &node.next { + let mut next_node: AddressNode = + serde_json::from_slice(&self.db.get_cf(&col, next_addr.as_bytes())?.unwrap()).unwrap(); + next_node.previous = node.previous; + batch.put_cf(&col, next_addr.as_bytes(), serde_json::to_vec(&next_node).unwrap()); + } else { + // Deleting tail + meta.tail = node.previous; + } + + meta.length -= 1; + + // If the deleted node was the one being iterated over, update the next iteration pointer + if let Some(next_iter_addr) = meta.next_iter_addr.as_ref() { + if next_iter_addr == &address { + meta.next_iter_addr = node.next; + } + } + + batch.delete_cf(&col, address.as_bytes()); + batch.put_cf(&col, METADATA_KEY, serde_json::to_vec(&meta).unwrap()); + + // Delete the node + // Updated previous node to point to next node + // Decremented metadata length by 1 + + return self.db.write(batch); + } + + Ok(()) // Address not found + } + + pub fn game_get_all_bot_addresses(&self) -> Result, rocksdb::Error> { + let col = self.db.get_column(Column::Game); + let meta = self.game_get_metadata()?; + let mut addresses = Vec::new(); + + let mut current = meta.head; + while let Some(addr) = current { + addresses.push(addr.clone()); + let node: AddressNode = serde_json::from_slice(&self.db.get_cf(&col, addr.as_bytes())?.unwrap()).unwrap(); + current = node.next; + } + + assert!( + addresses.len() == meta.length as usize, + "fetched addresses list != stored addresses length in metadata" + ); + + Ok(addresses) + } + + pub fn game_get_bots_list(&self) -> Result, rocksdb::Error> { + let mines_per_transaction = + env::var("MADARA_GAME_TPS").expect("MADARA_GAME_TPS not set").parse::().unwrap(); + + let col = self.db.get_column(Column::Game); + let meta = self.game_get_metadata()?; + + let next_start_address = meta.next_iter_addr; + + // Handle empty list case + if meta.length == 0 { + return Ok(vec![]); + } + let mut return_bots = Vec::with_capacity(mines_per_transaction as usize); + + // Start from the given address or head if None + let mut current_address = match next_start_address { + Some(addr) => Some(addr), + None => meta.head.clone(), + }; + + // Keep adding bots until we reach mines_per_transaction + while return_bots.len() < mines_per_transaction as usize { + if let Some(addr) = ¤t_address { + // Add current address + return_bots.push(addr.clone()); + + // Get next address, if we reach end, start from head + let node: AddressNode = + serde_json::from_slice(&self.db.get_cf(&col, addr.as_bytes())?.unwrap()).unwrap(); + + current_address = match node.next { + Some(next) => Some(next), + None => meta.head.clone(), // Wrap around to start + }; + } else { + break; // Should never happen if meta.length > 0 + } + } + + let mut meta = self.game_get_metadata()?; + // Update metadata with next start address + meta.next_iter_addr = current_address; + self.db.put_cf(&col, METADATA_KEY, serde_json::to_vec(&meta).unwrap())?; + // The next start address will be current_address + + Ok(return_bots) + } +} diff --git a/crates/madara/client/db/src/lib.rs b/crates/madara/client/db/src/lib.rs index 33bc43c4ee..1993bdf82f 100644 --- a/crates/madara/client/db/src/lib.rs +++ b/crates/madara/client/db/src/lib.rs @@ -34,6 +34,7 @@ pub mod contract_db; pub mod db_block_id; pub mod db_metrics; pub mod devnet_db; +pub mod game_db; pub mod l1_db; pub mod mempool_db; pub mod storage_updates; @@ -154,6 +155,8 @@ pub enum Column { Devnet, MempoolTransactions, + /// Game DB for Gridy + Game, } impl fmt::Debug for Column { @@ -168,6 +171,7 @@ impl fmt::Display for Column { } } +// TODO: why do I need to explicitly add Column memeber here, can't we fetch it dynamically from Column impl Column { pub const ALL: &'static [Self] = { use Column::*; @@ -201,6 +205,7 @@ impl Column { PendingContractStorage, Devnet, MempoolTransactions, + Game, ] }; pub const NUM_COLUMNS: usize = Self::ALL.len(); @@ -237,6 +242,7 @@ impl Column { PendingContractStorage => "pending_contract_storage", Devnet => "devnet", MempoolTransactions => "mempool_transactions", + Game => "game", } } } diff --git a/crates/madara/client/devnet/src/entrypoint.rs b/crates/madara/client/devnet/src/entrypoint.rs index 3a22ae2ebe..f58de9d528 100644 --- a/crates/madara/client/devnet/src/entrypoint.rs +++ b/crates/madara/client/devnet/src/entrypoint.rs @@ -21,6 +21,10 @@ impl Multicall { self } + pub fn with_vec(calls: Vec) -> Self { + Multicall(calls) + } + pub fn flatten(&self) -> impl Iterator + '_ { [self.0.len().into()].into_iter().chain(self.0.iter().flat_map(|c| c.flatten())) } diff --git a/crates/madara/primitives/chain_config/resources/versioned_constants_13_2.json b/crates/madara/primitives/chain_config/resources/versioned_constants_13_2.json index 4d3a9ceb9a..f29913cd27 100644 --- a/crates/madara/primitives/chain_config/resources/versioned_constants_13_2.json +++ b/crates/madara/primitives/chain_config/resources/versioned_constants_13_2.json @@ -2,7 +2,7 @@ "tx_event_limits": { "max_data_length": 300, "max_keys_length": 50, - "max_n_emitted_events": 1000 + "max_n_emitted_events": 5000 }, "gateway": { "max_calldata_length": 5000,