From f44438b0e12cc1e87a2a856fb6178dcb420bfd30 Mon Sep 17 00:00:00 2001 From: jennwiederholen Date: Fri, 25 Jul 2025 11:18:04 +0900 Subject: [PATCH 01/12] feat:initial-exex-indexer --- Cargo.lock | 215 ++++++++++++- Cargo.toml | 3 +- bin/searcher-reth/src/main.rs | 12 +- crates/extension/Cargo.toml | 3 + crates/extension/src/exex.rs | 100 +++++- crates/extension/src/lib.rs | 4 + crates/indexer/Cargo.toml | 48 +++ crates/indexer/config.yaml | 0 .../src/datasets/dolomite_borrow_position.rs | 65 ++++ crates/indexer/src/datasets/mod.rs | 1 + crates/indexer/src/db_writer.rs | 273 +++++++++++++++++ crates/indexer/src/indexer.rs | 289 ++++++++++++++++++ crates/indexer/src/lib.rs | 5 + crates/indexer/src/table_definitions.rs | 135 ++++++++ crates/indexer/src/utils.rs | 170 +++++++++++ crates/manager/src/common.rs | 2 + 16 files changed, 1313 insertions(+), 12 deletions(-) create mode 100644 crates/indexer/Cargo.toml create mode 100644 crates/indexer/config.yaml create mode 100644 crates/indexer/src/datasets/dolomite_borrow_position.rs create mode 100644 crates/indexer/src/datasets/mod.rs create mode 100644 crates/indexer/src/db_writer.rs create mode 100644 crates/indexer/src/indexer.rs create mode 100644 crates/indexer/src/lib.rs create mode 100644 crates/indexer/src/table_definitions.rs create mode 100644 crates/indexer/src/utils.rs diff --git a/Cargo.lock b/Cargo.lock index 263916a..2acacca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2921,6 +2921,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + [[package]] name = "fast-float2" version = "0.2.3" @@ -3969,6 +3975,24 @@ dependencies = [ "parity-scale-codec", ] +[[package]] +name = "impl-codec" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d40b9d5e17727407e55028eafc22b2dc68781786e6d7eb8a21103f5058e3a14" +dependencies = [ + "parity-scale-codec", +] + +[[package]] +name = "impl-serde" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a143eada6a1ec4aefa5049037a26a6d597bfd64f8c026d07b77133e02b7dd0b" +dependencies = [ + "serde", +] + [[package]] name = "impl-trait-for-tuples" version = "0.2.3" @@ -4748,6 +4772,16 @@ dependencies = [ "regex-automata 0.1.10", ] +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest 0.10.7", +] + [[package]] name = "memchr" version = "2.7.5" @@ -5512,6 +5546,36 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" +[[package]] +name = "postgres-protocol" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ff0abab4a9b844b93ef7b81f1efc0a366062aaef2cd702c76256b5dc075c54" +dependencies = [ + "base64 0.22.1", + "byteorder", + "bytes", + "fallible-iterator", + "hmac", + "md-5", + "memchr", + "rand 0.9.1", + "sha2 0.10.9", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613283563cd90e1dfc3518d548caee47e0e725455ed619881f5cf21f36de4b48" +dependencies = [ + "bytes", + "chrono", + "fallible-iterator", + "postgres-protocol", +] + [[package]] name = "potential_utf" version = "0.1.2" @@ -5562,10 +5626,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b34d9fd68ae0b74a41b21c03c2f62847aa0ffea044eee893b4c140b37e244e2" dependencies = [ "fixed-hash", - "impl-codec", + "impl-codec 0.6.0", "uint 0.9.5", ] +[[package]] +name = "primitive-types" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d15600a7d856470b7d278b3fe0e311fe28c2526348549f8ef2ff7db3299c87f5" +dependencies = [ + "fixed-hash", + "impl-codec 0.7.1", + "impl-serde", + "uint 0.10.0", +] + [[package]] name = "proc-macro-crate" version = "3.3.0" @@ -8939,7 +9015,7 @@ dependencies = [ "num-integer", "num-traits", "parity-scale-codec", - "primitive-types", + "primitive-types 0.12.2", "proptest", "rand 0.8.5", "rand 0.9.1", @@ -9299,9 +9375,50 @@ dependencies = [ "reth-revm", "reth-tracing", "reth-transaction-pool", + "searcher-reth-indexer", "searcher-reth-manager", "searcher-reth-strategy", "tokio", + "tokio-postgres", +] + +[[package]] +name = "searcher-reth-indexer" +version = "0.0.0" +dependencies = [ + "alloy-consensus", + "alloy-eips", + "alloy-genesis", + "alloy-network", + "alloy-primitives", + "alloy-rlp", + "alloy-rpc-types", + "alloy-signer-local", + "alloy-sol-types", + "chrono", + "eyre", + "futures", + "futures-util", + "lazy_static", + "primitive-types 0.13.1", + "reth", + "reth-chainspec", + "reth-evm", + "reth-execution-errors", + "reth-execution-types", + "reth-exex", + "reth-node-api", + "reth-node-ethereum", + "reth-primitives", + "reth-provider", + "reth-revm", + "reth-tracing", + "reth-transaction-pool", + "serde", + "serde_yaml", + "tokio", + "tokio-postgres", + "tracing", ] [[package]] @@ -9562,6 +9679,19 @@ dependencies = [ "syn 2.0.104", ] +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap 2.10.0", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "serdect" version = "0.2.0" @@ -9826,6 +9956,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.11.1" @@ -10216,6 +10357,32 @@ dependencies = [ "syn 2.0.104", ] +[[package]] +name = "tokio-postgres" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c95d533c83082bb6490e0189acaa0bbeef9084e60471b696ca6988cd0541fb0" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator", + "futures-channel", + "futures-util", + "log", + "parking_lot", + "percent-encoding", + "phf", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "rand 0.9.1", + "socket2", + "tokio", + "tokio-util", + "whoami", +] + [[package]] name = "tokio-rustls" version = "0.24.1" @@ -10618,12 +10785,33 @@ version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +[[package]] +name = "unicode-normalization" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" + [[package]] name = "unicode-segmentation" version = "1.12.0" @@ -10669,6 +10857,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "unsigned-varint" version = "0.8.0" @@ -10830,6 +11024,12 @@ dependencies = [ "wit-bindgen-rt", ] +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" version = "0.2.100" @@ -10990,6 +11190,17 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "whoami" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6994d13118ab492c3c80c1f81928718159254c53c472bf9ce36f8dae4add02a7" +dependencies = [ + "redox_syscall", + "wasite", + "web-sys", +] + [[package]] name = "widestring" version = "1.2.0" diff --git a/Cargo.toml b/Cargo.toml index 8c28643..a06ea2d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["bin/searcher-reth", "crates/extension", "crates/manager", "crates/strategy"] +members = ["bin/searcher-reth", "crates/extension", "crates/manager", "crates/strategy", "crates/indexer"] resolver = "2" [workspace.package] version = "0.1.0" @@ -14,6 +14,7 @@ searcher-reth = { path = "bin/searcher-reth" } searcher-reth-extension = { path = "crates/extension" } searcher-reth-manager = { path = "crates/manager" } searcher-reth-strategy = { path = "crates/strategy" } +searcher-reth-indexer = { path = "crates/indexer"} # reth reth = { git = "https://github.com/paradigmxyz/reth", tag = "v1.5.1" } diff --git a/bin/searcher-reth/src/main.rs b/bin/searcher-reth/src/main.rs index 339e787..48fcd7b 100644 --- a/bin/searcher-reth/src/main.rs +++ b/bin/searcher-reth/src/main.rs @@ -5,14 +5,14 @@ use reth::chainspec::EthereumChainSpecParser; use reth_node_ethereum::EthereumNode; use reth_tracing::tracing::error; use searcher_reth_extension::{ - exex::SearcherExEx, + exex::{IndexerExEx, SearcherExEx}, relayer_pool::WalletPool, strategy::{ core::strategy::Strategy, path_finding::PathFinder, profit_reporter::init_reporter, }, util::signal_manager::SignalManager, }; -use searcher_reth_manager::{common::PATH_FINDER_EXEX_ID, manager::ConfigManager}; +use searcher_reth_manager::{common::{INDEXER_EXEX_ID, PATH_FINDER_EXEX_ID}, manager::ConfigManager}; fn main() -> eyre::Result<()> { let config = Arc::new(RwLock::new(ConfigManager::from_file("env.toml")?)); @@ -35,6 +35,9 @@ fn main() -> eyre::Result<()> { // Install Exex for various strategies let searcher_exex = SearcherExEx::new(wallet, signal_manager.subscribe()); + // Init Exex for Indexer + let indexer_exex = IndexerExEx::new(); + let mut node_builder = builder.node(EthereumNode::default()); // Install PathFinder strategy @@ -45,6 +48,11 @@ fn main() -> eyre::Result<()> { move |ctx| searcher_exex.run(ctx, path_finder) }); + // Install Indexer + node_builder = node_builder.install_exex(INDEXER_EXEX_ID, { + IndexerExEx::init + }); + // TODO: Add other strategies here as needed let handle = node_builder.launch().await?; handle.wait_for_node_exit().await diff --git a/crates/extension/Cargo.toml b/crates/extension/Cargo.toml index 2fcbfd9..dbc31db 100644 --- a/crates/extension/Cargo.toml +++ b/crates/extension/Cargo.toml @@ -42,7 +42,10 @@ clap.workspace = true searcher-reth-strategy.workspace = true searcher-reth-manager.workspace = true +searcher-reth-indexer.workspace = true rayon.workspace = true reth-transaction-pool.workspace = true +# postgres +tokio-postgres = { version = "0.7.13", features = ["with-chrono-0_4"] } \ No newline at end of file diff --git a/crates/extension/src/exex.rs b/crates/extension/src/exex.rs index 706b098..fb830d6 100644 --- a/crates/extension/src/exex.rs +++ b/crates/extension/src/exex.rs @@ -1,21 +1,107 @@ use std::{future::Future, sync::Arc}; -use eyre::Result; -use futures_util::StreamExt; +use eyre::{Result, WrapErr}; +use futures_util::{StreamExt, TryStreamExt}; use reth::network::NetworkInfo; -use reth_exex::{ExExContext, ExExEvent, ExExNotification}; -use reth_node_api::{FullNodeComponents, FullNodeTypes}; +use reth::builder::NodeTypes; +use reth_exex::{ ExExContext, ExExEvent, ExExNotification }; +use reth_node_api::{ FullNodeComponents, FullNodeTypes }; use reth_provider::{ AccountReader, BlockHashReader, BlockReaderIdExt, ChainSpecProvider, DatabaseProviderFactory, LatestStateProviderRef, ReceiptProvider, StateCommitmentProvider, }; -use reth_tracing::tracing::{self}; -use reth_transaction_pool::{EthPooledTransaction, TransactionPool}; +use reth_primitives::{TransactionSigned, Receipt}; +use reth_tracing::tracing::{ self }; +use reth_transaction_pool::{ EthPooledTransaction, TransactionPool }; use searcher_reth_manager::SignalType; use searcher_reth_strategy::core::strategy::Strategy; use tokio::sync::broadcast; +use crate::relayer_pool::{ RelayerMessage, RelayerPool, WalletPool }; +use searcher_reth_indexer::{indexer::Indexer, db_writer::DbWriter }; +use alloy_consensus::{BlockHeader, Header, Block}; +use reth::primitives::{EthereumHardforks, NodePrimitives}; +use searcher_reth_indexer::utils::{Config, connect_to_postgres, create_tables}; +use tokio_postgres::Client; -use crate::relayer_pool::{RelayerMessage, RelayerPool, WalletPool}; +pub struct IndexerExEx {} + +impl IndexerExEx { + pub fn new() -> Self { + Self {} + } + + pub async fn init( + ctx: ExExContext, + ) -> Result>> + where + Node::Types: NodeTypes, + ::ChainSpec: EthereumHardforks, + ::Primitives: NodePrimitives< + BlockHeader = Header, + Block = Block, + Receipt = Receipt, + SignedTx = TransactionSigned + >, + { + let config = Config::load().wrap_err("Failed to load configuration")?; + + let client = Arc::new(connect_to_postgres().await?); + create_tables(&client).await?; + + // Create indexer with all processors initialized internally + let indexer = Indexer::new(config); + + Ok(Self::indexer_exex(ctx, client, indexer)) + } + + async fn indexer_exex( + mut ctx: ExExContext, + client: Arc, + indexer: Indexer, + ) -> Result<()> + where + Node::Types: NodeTypes, + ::ChainSpec: EthereumHardforks, + ::Primitives: NodePrimitives< + BlockHeader = Header, + Block = Block, + Receipt = Receipt, + SignedTx = TransactionSigned + >, + { + // Process all new chain state notifications + while let Some(notification) = ctx.notifications.try_next().await? { + match ¬ification { + ExExNotification::ChainCommitted { new } => { + // Process the committed blocks + if let Err(e) = indexer.process_blocks( + new.blocks_and_receipts(), + &client, + ctx.provider().clone(), + Arc::new(ctx.evm_config().clone()), + Arc::new(ctx.pool().clone()), + Arc::new(ctx.network().clone()), + ).await { + tracing::warn!("Failed to process committed blocks: {}", e); + } + + // Advance ExEx + ctx.events.send(ExExEvent::FinishedHeight(new.tip().num_hash()))?; + }, + ExExNotification::ChainReorged { .. } => { + // do nothing for reorg + continue; + }, + ExExNotification::ChainReverted { .. } => { + // do nothing for revert + continue; + }, + } + } + Ok(()) + } + +} pub struct SearcherExEx { pub wallet: Arc, diff --git a/crates/extension/src/lib.rs b/crates/extension/src/lib.rs index 0b94732..2b625f2 100644 --- a/crates/extension/src/lib.rs +++ b/crates/extension/src/lib.rs @@ -8,3 +8,7 @@ pub mod util { pub mod strategy { pub use searcher_reth_strategy::*; } + +pub mod indexer { + pub use searcher_reth_indexer::*; +} diff --git a/crates/indexer/Cargo.toml b/crates/indexer/Cargo.toml new file mode 100644 index 0000000..74e15a3 --- /dev/null +++ b/crates/indexer/Cargo.toml @@ -0,0 +1,48 @@ +[package] +name = "searcher-reth-indexer" +version = "0.0.0" +publish = false +edition.workspace = true +license.workspace = true + +[dependencies] +# reth +reth-chainspec.workspace = true +reth-evm.workspace = true +reth-execution-errors.workspace = true +reth-execution-types.workspace = true +reth-exex.workspace = true +reth-node-api.workspace = true +reth-node-ethereum.workspace = true +reth-primitives.workspace = true +reth-provider.workspace = true +reth-revm.workspace = true +reth-tracing.workspace = true +reth.workspace = true + +# alloy +alloy-consensus = { workspace = true, features = ["k256"] } +alloy-eips.workspace = true +alloy-genesis.workspace = true +alloy-primitives.workspace = true +alloy-rlp.workspace = true +alloy-sol-types.workspace = true +alloy-network.workspace = true +alloy-signer-local.workspace = true +alloy-rpc-types.workspace = true + +# async +futures-util.workspace = true +tokio.workspace = true + +reth-transaction-pool.workspace = true + +tokio-postgres = { version = "0.7.13", features = ["with-chrono-0_4"] } +serde = { workspace = true, features = ["derive"] } +lazy_static = "1.5.0" +eyre.workspace = true +futures = "0.3.31" +chrono = "0.4.40" +serde_yaml = "0.9.34" +primitive-types = { version = "0.13.1", features = ["serde"] } +tracing = "0.1.41" \ No newline at end of file diff --git a/crates/indexer/config.yaml b/crates/indexer/config.yaml new file mode 100644 index 0000000..e69de29 diff --git a/crates/indexer/src/datasets/dolomite_borrow_position.rs b/crates/indexer/src/datasets/dolomite_borrow_position.rs new file mode 100644 index 0000000..26340a9 --- /dev/null +++ b/crates/indexer/src/datasets/dolomite_borrow_position.rs @@ -0,0 +1,65 @@ +use crate::record_values; +use crate::db_writer::DbWriter; +use alloy_sol_types::{sol, SolEvent}; +use alloy_primitives::{address, Address}; +use reth_primitives::{RecoveredBlock, Receipt, Block}; +use chrono::Utc; +use eyre::Result; +use reth_node_api::FullNodeComponents; +use tracing::debug; +use crate::indexer::ProcessingComponents; +// BorrowPositionProxyV2 address +const DOLOMITE_CONTRACT_ADDRESS: Address = address!("0xC06271eb97d960F4034DDF953e16271CcB2B10BD"); + +sol! { + event BorrowPositionOpen( + address indexed _borrower, + uint256 indexed _borrowAccountNumber + ); +} + +pub async fn process_dolomite_borrow_positions( + block_data: &(RecoveredBlock, Vec), + components: ProcessingComponents, + writer: &mut DbWriter +) -> Result<()> { + let block = &block_data.0; + let receipts = &block_data.1; + let block_number = block.number; + + // Iterate through transactions and their receipts + for (tx_idx, (tx, receipt)) in block.body().transactions.iter().zip(receipts.iter()).enumerate() { + // Process each log in the receipt + for (log_idx, log) in receipt.logs.iter().enumerate() { + // Filter on dolomite contract address + if log.address == DOLOMITE_CONTRACT_ADDRESS { + // First check if this log matches our event signature + if log.topics().get(0) != Some(&BorrowPositionOpen::SIGNATURE_HASH) { + continue; + } + + match BorrowPositionOpen::decode_raw_log(log.topics(), &log.data.data) { + Ok(create) => { + writer + .write_record(record_values![ + block_number as i64, + tx.hash(), + tx_idx as i64, + log_idx as i64, + log.address, + create._borrower, + create._borrowAccountNumber, + Utc::now(), + ]) + .await?; + } + Err(e) => { + debug!("Failed to decode dolomite borrow position open event: {:?}", e); + } + } + } + } + } + Ok(()) +} + diff --git a/crates/indexer/src/datasets/mod.rs b/crates/indexer/src/datasets/mod.rs new file mode 100644 index 0000000..c1c25cd --- /dev/null +++ b/crates/indexer/src/datasets/mod.rs @@ -0,0 +1 @@ +pub mod dolomite_borrow_position; \ No newline at end of file diff --git a/crates/indexer/src/db_writer.rs b/crates/indexer/src/db_writer.rs new file mode 100644 index 0000000..2a207bd --- /dev/null +++ b/crates/indexer/src/db_writer.rs @@ -0,0 +1,273 @@ +use crate::table_definitions::TableDefinition; +use eyre::Result; +use std::sync::Arc; +use tokio_postgres::{Client, Statement, types::Type}; +use futures::pin_mut; +use tokio_postgres::binary_copy::BinaryCopyInWriter; +use alloy_primitives::{Address, FixedBytes, Uint, Signed}; + +pub struct DbWriter { + client: Arc, + revert_stmt: Statement, + table: TableDefinition, + records: Vec>>, +} + +impl DbWriter { + pub async fn new(client: &Arc, table: TableDefinition) -> Result { + let revert_stmt = client + .prepare(&table.revert_statement()) + .await?; + + Ok(Self { + client: Arc::clone(client), + revert_stmt, + table, + records: Vec::new(), + }) + } + + pub async fn write_record(&mut self, record: Vec>) -> Result<()> { + self.records.push(record); + Ok(()) + } + + pub async fn finish(self) -> Result { + if self.records.is_empty() { + return Ok(0); + } + + let column_names = self.table.columns.iter() + .map(|col| col.name) + .collect::>() + .join(", "); + + // Start COPY operation + let copy_stmt = format!( + "COPY {} ({}) FROM STDIN BINARY", + self.table.name, + column_names + ); + + let sink = self.client.copy_in(©_stmt).await?; + + // Create a binary writer with the correct types + let writer = BinaryCopyInWriter::new( + sink, + &self.table.columns.iter() + .map(|col| col.get_postgres_type()) + .collect::>() + ); + pin_mut!(writer); + + // Process all records + let mut total_records = 0; + for record in self.records { + // Convert the boxed ToSql traits to references + let record_refs: Vec<&(dyn tokio_postgres::types::ToSql + Sync)> = record + .iter() + .map(|value| value.as_ref() as &(dyn tokio_postgres::types::ToSql + Sync)) + .collect(); + + writer.as_mut().write(&record_refs).await?; + total_records += 1; + } + + // Finish the COPY operation + writer.finish().await?; + + Ok(total_records) + } + + pub async fn revert(&self, block_numbers: &[i64]) -> Result<()> { + self.client + .execute(&self.revert_stmt, &[&block_numbers]) + .await?; + Ok(()) + } +} + +// Helper trait to make it easier to convert common Ethereum data types to Postgres types +pub trait EthereumValue { + fn to_sql_value(&self) -> Box; +} + +impl EthereumValue for primitive_types::H256 { + fn to_sql_value(&self) -> Box { + Box::new(self.to_string()) + } +} + +impl EthereumValue for primitive_types::U256 { + fn to_sql_value(&self) -> Box { + Box::new(self.to_string()) + } +} + +impl EthereumValue for alloy_primitives::Address { + fn to_sql_value(&self) -> Box { + Box::new(self.to_checksum(Some(1))) + } +} + +impl EthereumValue for Option { + fn to_sql_value(&self) -> Box { + Box::new(self.clone()) + } +} + +impl<'a> EthereumValue for Option<&'a str> { + fn to_sql_value(&self) -> Box { + Box::new(self.map(|s| s.to_string())) + } +} + +impl EthereumValue for Option { + fn to_sql_value(&self) -> Box { + Box::new(self.clone()) + } +} + +impl EthereumValue for Option { + fn to_sql_value(&self) -> Box { + Box::new(self.map(|v| v as i64)) + } +} + +impl EthereumValue for Option { + fn to_sql_value(&self) -> Box { + Box::new(self.clone()) + } +} + +impl EthereumValue for Option> { + fn to_sql_value(&self) -> Box { + Box::new(self.clone()) + } +} + +impl EthereumValue for Option { + fn to_sql_value(&self) -> Box { + Box::new(self.as_ref().map(|addr| addr.to_checksum(Some(1)))) + } +} + +impl EthereumValue for Option { + fn to_sql_value(&self) -> Box { + Box::new(self.as_ref().map(|h| h.to_string())) + } +} + +impl EthereumValue for Option { + fn to_sql_value(&self) -> Box { + Box::new(self.as_ref().map(|u| u.to_string())) + } +} + +impl EthereumValue for Option> { + fn to_sql_value(&self) -> Box { + Box::new(self.as_ref().map(|bytes| bytes.to_string())) + } +} + +impl<'a> EthereumValue for Option<&'a alloy_primitives::FixedBytes<32>> { + fn to_sql_value(&self) -> Box { + Box::new(self.map(|bytes| bytes.to_string())) + } +} + +impl EthereumValue for Option> { + fn to_sql_value(&self) -> Box { + Box::new(self.as_ref().map(|u| u.to_string())) + } +} + +impl EthereumValue for Vec { + fn to_sql_value(&self) -> Box { + Box::new(alloy_primitives::hex::encode_prefixed(self)) + } +} + +impl EthereumValue for [u8] { + fn to_sql_value(&self) -> Box { + Box::new(alloy_primitives::hex::encode_prefixed(self)) + } +} + +impl EthereumValue for String { + fn to_sql_value(&self) -> Box { + Box::new(self.clone()) + } +} + +impl EthereumValue for str { + fn to_sql_value(&self) -> Box { + Box::new(self.to_string()) + } +} + +impl EthereumValue for i64 { + fn to_sql_value(&self) -> Box { + Box::new(*self) + } +} + +impl EthereumValue for i32 { + fn to_sql_value(&self) -> Box { + Box::new(*self as i64) + } +} + +impl EthereumValue for bool { + fn to_sql_value(&self) -> Box { + Box::new(*self) + } +} + +impl EthereumValue for chrono::DateTime { + fn to_sql_value(&self) -> Box { + Box::new(*self) + } +} + +impl EthereumValue for alloy_primitives::FixedBytes<32> { + fn to_sql_value(&self) -> Box { + Box::new(self.to_string()) + } +} + +impl EthereumValue for alloy_primitives::Uint { + fn to_sql_value(&self) -> Box { + Box::new(self.to_string()) + } +} + +impl EthereumValue for alloy_primitives::Signed { + fn to_sql_value(&self) -> Box { + Box::new(self.to_string()) + } +} + +// Implement for references to types that implement EthereumValue +impl EthereumValue for &T { + fn to_sql_value(&self) -> Box { + (*self).to_sql_value() + } +} + +impl EthereumValue for f64 { + fn to_sql_value(&self) -> Box { + Box::new(*self) + } +} + +// Macro to help convert datatypes to Postgres types (via EthereumValue trait implementations) +#[macro_export] +macro_rules! record_values { + ($($value:expr),* $(,)?) => {{ + // Import here so it does not need to be imported in dataset files + use $crate::db_writer::EthereumValue; + let values: Vec> = vec![$($value.to_sql_value()),*]; + values + }}; +} \ No newline at end of file diff --git a/crates/indexer/src/indexer.rs b/crates/indexer/src/indexer.rs new file mode 100644 index 0000000..db05fa1 --- /dev/null +++ b/crates/indexer/src/indexer.rs @@ -0,0 +1,289 @@ +use crate::utils::Config; +use crate::table_definitions::get_table; +use crate::db_writer::DbWriter; +use crate::datasets::{ + dolomite_borrow_position::process_dolomite_borrow_positions +}; +use alloy_consensus::{Header}; +use alloy_rpc_types::{BlockId, BlockNumberOrTag}; +// use alloy_rpc_types_trace::parity::{TraceResultsWithTransactionHash, TraceType}; +use eyre::Result; +use reth::builder::NodeTypes; +use reth::primitives::{EthereumHardforks, NodePrimitives}; +use reth_primitives::{TransactionSigned, Receipt, RecoveredBlock, Block}; +use reth_node_api::{ConfigureEvm, FullNodeComponents, FullNodeTypes}; +// use reth_rpc::EthApi; +// use reth_rpc_eth_api::{FullEthApiTypes, helpers::{Call, LoadPendingBlock}}; +use reth_tracing::tracing::{info, warn}; +use std::{sync::Arc, time::Instant, collections::HashSet}; +use tokio_postgres::Client; + +// Structure to hold all the components needed for processing +#[derive(Clone)] +pub struct ProcessingComponents { + // pub eth_api: Arc>, + // pub block_traces: Option>, + pub provider: Node::Provider, + pub client: Arc, + pub config: Config, +} + +struct ProcessorInfo { + table_name: &'static str, + processor_name: &'static str, + processor: for<'a> fn( + &'a (RecoveredBlock, Vec), + ProcessingComponents, + &'a mut DbWriter + ) -> futures::future::BoxFuture<'a, Result<()>>, +} + +impl ProcessorInfo { + fn new( + table_name: &'static str, + processor_name: &'static str, + processor: for<'a> fn( + &'a (RecoveredBlock, Vec), + ProcessingComponents, + &'a mut DbWriter + ) -> futures::future::BoxFuture<'a, Result<()>>, + ) -> Self { + Self { + table_name, + processor_name, + processor, + } + } +} + +pub struct Indexer { + processors: Vec>, + config: Config, +} + +impl Indexer +where + Node: FullNodeComponents + FullNodeTypes, + Node::Types: NodeTypes, + ::ChainSpec: EthereumHardforks, + ::Primitives: NodePrimitives< + BlockHeader = Header, + Block = Block, + Receipt = Receipt, + SignedTx = TransactionSigned, + >, + Node::Provider: reth::providers::BlockReader> + + reth::providers::HeaderProvider
+ + reth::providers::ReceiptProvider + + reth::providers::TransactionsProvider, + Node::Evm: ConfigureEvm + //EthApi: Call + LoadPendingBlock + FullEthApiTypes, +{ + pub fn new(config: Config) -> Self { + let mut indexer = Self { + processors: Vec::new(), + config, + }; + + // Register all available processors + // e.g) indexer.add_processor("headers", "Headers"); + indexer.add_processor("dolomite_borrow_positions", "DolomiteBorrowPositions"); + + info!("Initialized indexer with processors: {:?}", indexer.list_processors()); + indexer + } + + pub fn add_processor(&mut self, table_name: &'static str, processor_name: &'static str) { + let processor = match table_name { + "dolomite_borrow_positions" => ProcessorInfo::new( + table_name, + processor_name, + |block_data, components, writer| Box::pin(process_dolomite_borrow_positions::(block_data, components, writer)) + ), + _ => return, // Skip unknown processors + }; + self.processors.push(processor); + } + + pub fn list_processors(&self) -> Vec<&str> { + self.processors.iter().map(|p| p.processor_name).collect() + } + + pub async fn process_blocks( + &self, + blocks_and_receipts: impl Iterator, &Vec)>, + client: &Arc, + provider: Node::Provider, + evm_config: Arc, + pool: Arc, + network: Arc, + ) -> Result<()> + where + Node::Evm: ConfigureEvm + //EthApi: Call + LoadPendingBlock + FullEthApiTypes, + { + // Convert the iterator items into owned values directly + let blocks_and_receipts: Vec<_> = blocks_and_receipts + .map(|(block, receipts)| (block.clone(), receipts.clone())) + .collect(); + + for (block, receipts) in blocks_and_receipts { + let block_number = block.number; + let block_id = BlockId::Number(BlockNumberOrTag::from(block_number)); + + // Create EthAPI + // let eth_api = crate::utils::create_eth_api::( + // provider.clone(), + // (*evm_config).clone(), + // (*pool).clone(), + // (*network).clone() + // ); + + // // Create TraceAPI + // let trace_api = crate::utils::create_trace_api::( + // provider.clone(), + // (*evm_config).clone(), + // (*pool).clone(), + // (*network).clone() + // ); + + // // Get traces once for the block + // let block_traces = match trace_api.replay_block_transactions( + // block_id, + // HashSet::from_iter(vec![TraceType::Trace]) + // ).await { + // Ok(traces) => traces, + // Err(e) => { + // warn!("Failed to get traces for block {}: {}", block_number, e); + // None + // } + // }; + + // Create components for processing + let components = ProcessingComponents { + // eth_api: eth_api.clone(), + // block_traces, + provider: provider.clone(), + client: Arc::clone(client), + config: self.config.clone(), + }; + + let block_data = (block, receipts); + if let Err(e) = self.process_block_data(&block_data, components).await { + warn!("Failed to process block {}: {}", block_number, e); + } + } + + Ok(()) + } + + pub async fn process_block_data( + &self, + block_data: &(RecoveredBlock, Vec), + components: ProcessingComponents, + ) -> Result<()> + where + Node::Types: NodeTypes, + ::ChainSpec: EthereumHardforks, + ::Primitives: NodePrimitives< + BlockHeader = Header, + Block = Block, + Receipt = Receipt, + SignedTx = TransactionSigned + >, + { + let block_number = block_data.0.number; + + // Create a vector to store all processing tasks + let mut tasks = Vec::new(); + + // Spawn a task for each enabled processor + for processor in &self.processors { + if self.config.is_event_enabled(processor.processor_name) { + // Clone necessary data for the task + let processor_name = processor.processor_name; + let processor_fn = processor.processor; + let block_data = block_data.clone(); + let components = components.clone(); + let table = get_table(processor.table_name) + .expect(&format!("Table definition not found for {}", processor.table_name)); + + // Spawn the task + let task = tokio::spawn(async move { + let event_start_time = Instant::now(); + let mut writer = match DbWriter::new(&components.client, table).await { + Ok(w) => w, + Err(e) => return Err((processor_name, e.to_string())) + }; + match processor_fn(&block_data, components, &mut writer).await { + Ok(()) => { + match writer.finish().await { + Ok(records_written) => Ok((processor_name, records_written, event_start_time.elapsed())), + Err(e) => Err((processor_name, e.to_string())) + } + }, + Err(e) => Err((processor_name, e.to_string())) + } + }); + + tasks.push(task); + } + } + + // Wait for all tasks to complete and collect results + let mut total_records = 0; + let mut event_results = Vec::new(); + let mut failed_events = Vec::new(); + + for task in tasks { + match task.await { + Ok(Ok((name, records, duration))) => { + total_records += records; + event_results.push((name, records, duration)); + } + Ok(Err((name, error))) => { + failed_events.push((name, error)); + } + Err(e) => { + warn!("Task join error: {}", e); + } + } + } + + // Sort events by name for consistent logging + event_results.sort_by(|a, b| a.0.cmp(&b.0)); + + // Create a consolidated success log + if !event_results.is_empty() { + let events_summary: Vec = event_results + .iter() + .map(|(name, records, time)| { + format!("{}({}, {:.2}s)", name, records, time.as_secs_f64()) + }) + .collect(); + + info!( + "exex{{id=\"exex-indexer\"}}: Block {} processed - Events: [{}], Total records: {}", + block_number, + events_summary.join(", "), + total_records, + ); + } + + // Create a consolidated error log + if !failed_events.is_empty() { + let failure_summary: Vec = failed_events + .iter() + .map(|(name, error)| format!("{}: {}", name, error)) + .collect(); + + warn!( + "exex{{id=\"exex-indexer\"}}: Block {} failures - {}", + block_number, + failure_summary.join(", ") + ); + } + + Ok(()) + } +} \ No newline at end of file diff --git a/crates/indexer/src/lib.rs b/crates/indexer/src/lib.rs new file mode 100644 index 0000000..a3bdc97 --- /dev/null +++ b/crates/indexer/src/lib.rs @@ -0,0 +1,5 @@ +pub mod indexer; +pub mod db_writer; +pub mod table_definitions; +pub mod utils; +pub mod datasets; \ No newline at end of file diff --git a/crates/indexer/src/table_definitions.rs b/crates/indexer/src/table_definitions.rs new file mode 100644 index 0000000..cbda165 --- /dev/null +++ b/crates/indexer/src/table_definitions.rs @@ -0,0 +1,135 @@ +use lazy_static::lazy_static; +use tokio_postgres::types::Type; + +#[derive(Debug, Clone)] +pub struct TableDefinition { + pub name: &'static str, + pub columns: Vec, + pub indexes: Vec<&'static str>, +} + +#[derive(Debug, Clone)] +pub struct ColumnDefinition { + pub name: &'static str, + pub sql_type: &'static str, + pub nullable: bool, + pub primary_key: bool, +} + +impl ColumnDefinition { + pub fn get_postgres_type(&self) -> Type { + match self.sql_type { + "BIGINT" => Type::INT8, + "INTEGER" => Type::INT4, + "SMALLINT" => Type::INT2, + "TEXT" | "VARCHAR" => Type::TEXT, + "BOOLEAN" => Type::BOOL, + "DOUBLE PRECISION" => Type::FLOAT8, + "REAL" => Type::FLOAT4, + "TIMESTAMP WITH TIME ZONE" => Type::TIMESTAMPTZ, + "TIMESTAMP" => Type::TIMESTAMP, + "DATE" => Type::DATE, + "BYTEA" => Type::BYTEA, + "JSON" => Type::JSON, + "JSONB" => Type::JSONB, + _ => Type::TEXT, // Default to TEXT for unknown types + } + } +} + +impl TableDefinition { + pub fn create_table_sql(&self) -> String { + let columns: Vec = self.columns + .iter() + .map(|col| { + let nullable = if col.nullable { "" } else { " NOT NULL" }; + let pk = if col.primary_key { " PRIMARY KEY" } else { "" }; + format!("{} {}{}{}", col.name, col.sql_type, nullable, pk) + }) + .collect(); + + format!( + "CREATE TABLE IF NOT EXISTS {} (\n {}\n)", + self.name, + columns.join(",\n ") + ) + } + + pub fn create_index_statements(&self) -> Vec { + self.indexes.iter().map(|idx| idx.to_string()).collect() + } + + pub fn revert_statement(&self) -> String { + format!("DELETE FROM {} WHERE block_number = ANY($1::bigint[])", self.name) + } +} + +// Define all table schemas +pub fn get_table_definitions() -> Vec { + vec![ + TableDefinition { + name: "dolomite_borrow_positions", + columns: vec![ + ColumnDefinition { + name: "block_number", + sql_type: "BIGINT", + nullable: false, + primary_key: false, + }, + ColumnDefinition { + name: "transaction_hash", + sql_type: "VARCHAR", + nullable: false, + primary_key: false, + }, + ColumnDefinition { + name: "transaction_index", + sql_type: "BIGINT", + nullable: false, + primary_key: false, + }, + ColumnDefinition { + name: "log_index", + sql_type: "BIGINT", + nullable: false, + primary_key: false, + }, + ColumnDefinition { + name: "log_address", + sql_type: "VARCHAR", + nullable: false, + primary_key: false, + }, + ColumnDefinition { + name: "borrower", + sql_type: "VARCHAR", + nullable: false, + primary_key: false, + }, + ColumnDefinition { + name: "borrower_number", + sql_type: "BIGINT", + nullable: false, + primary_key: false, + }, + ], + indexes: vec![ + "CREATE INDEX IF NOT EXISTS idx_dolomite_borrow_positions_borrower ON dolomite_borrow_positions (borrower)", + ], + }, + ] +} + +// Lazy static reference to table definitions for easy access +lazy_static! { + pub static ref TABLES: Vec = get_table_definitions(); +} + +// Helper function to get a specific table definition +pub fn get_table_definition(name: &str) -> Option { + TABLES.iter().find(|t| t.name == name).cloned() +} + +pub fn get_table(name: &str) -> Option { + get_table_definition(name) +} \ No newline at end of file diff --git a/crates/indexer/src/utils.rs b/crates/indexer/src/utils.rs new file mode 100644 index 0000000..4cd85e2 --- /dev/null +++ b/crates/indexer/src/utils.rs @@ -0,0 +1,170 @@ +use crate::table_definitions::TABLES; +// use alloy_eips::eip1559::ETHEREUM_BLOCK_GAS_LIMIT; +// use reth::builder::NodeTypes; +// use reth_node_api::FullNodeComponents; +// use reth_rpc::{EthApi, TraceApi}; +// use reth_rpc_eth_api::helpers::{Call, LoadPendingBlock}; +// use reth_rpc_eth_types::{EthStateCache, GasPriceOracle, FeeHistoryCache, FeeHistoryCacheConfig}; +// use reth_rpc_server_types::constants::{ +// DEFAULT_ETH_PROOF_WINDOW, +// DEFAULT_MAX_SIMULATE_BLOCKS, +// DEFAULT_PROOF_PERMITS, +// }; +// use reth_tasks::pool::{BlockingTaskGuard, BlockingTaskPool}; +use reth_tracing::tracing::info; +use serde::{Deserialize, Serialize}; +use std::{sync::Arc, collections::HashMap, env}; +use tokio_postgres::{Client, NoTls}; + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct Config { + pub enabled_events: HashMap, + #[serde(flatten)] + pub dataset_configs: HashMap, +} + +impl Config { + pub fn load() -> eyre::Result { + // The path is relative to this file + let config_str = include_str!("../config.yaml"); + let config = serde_yaml::from_str(config_str)?; + Ok(config) + } + + pub fn is_event_enabled(&self, event_name: &str) -> bool { + self.enabled_events.get(event_name).cloned().unwrap_or(true) + } + + pub fn get(&self, key: &str) -> eyre::Result { + self.dataset_configs.get(key) + .ok_or_else(|| eyre::eyre!("No config found for {}", key)) + .and_then(|v| serde_yaml::from_value(v.clone()) + .map_err(|e| eyre::eyre!("Failed to parse config for {}: {}", key, e))) + } +} + +/// Sanitize a byte slice to ensure it's valid UTF-8. +pub(crate) fn sanitize_bytes(input: &[u8]) -> String { + // First, remove all null bytes + let without_nulls: Vec = input.iter().filter(|&&b| b != 0).cloned().collect(); + + // Then, convert to a string, replacing any invalid UTF-8 sequences + String::from_utf8_lossy(&without_nulls).into_owned() +} + +/// Connect to the Postgres database. +pub async fn connect_to_postgres() -> eyre::Result { + let db_url = env::var("DATABASE_URL").expect("DATABASE_URL must be set"); + let (client, connection) = tokio_postgres::connect(&db_url, NoTls).await?; + + tokio::spawn(async move { + if let Err(e) = connection.await { + eprintln!("Connection error: {}", e); + } + }); + + Ok(client) +} + +/// Create Postgres tables if they do not exist. +pub async fn create_tables(client: &Client) -> eyre::Result<()> { + for table in TABLES.iter() { + // Create the table + let create_table_sql = table.create_table_sql(); + client.execute(&create_table_sql, &[]).await?; + + // Create the indexes + for index_sql in table.create_index_statements() { + client.execute(&index_sql, &[]).await?; + } + } + + info!("Initialized database tables"); + Ok(()) +} + +// Create a trace API instance with all the required components and trait bounds +// pub fn create_trace_api( +// provider: Node::Provider, +// evm_config: Node::Evm, +// pool: Node::Pool, +// network: Node::Network, +// ) -> Arc>> +// where +// Node: FullNodeComponents, +// Node::Types: NodeTypes, +// EthApi: Call + LoadPendingBlock, +// { +// let cache = EthStateCache::spawn( +// provider.clone(), +// Default::default(), +// ); + +// let fee_history_cache = FeeHistoryCache::new(FeeHistoryCacheConfig::default()); + +// let gas_oracle = GasPriceOracle::new( +// provider.clone(), +// Default::default(), +// cache.clone() +// ); + +// let eth_api = EthApi::new( +// provider.clone(), +// pool, +// network, +// cache.clone(), +// gas_oracle, +// ETHEREUM_BLOCK_GAS_LIMIT, +// DEFAULT_MAX_SIMULATE_BLOCKS, +// DEFAULT_ETH_PROOF_WINDOW, +// BlockingTaskPool::build().expect("failed to build tracing pool"), +// fee_history_cache, +// evm_config.clone(), +// DEFAULT_PROOF_PERMITS, +// ); + +// Arc::new(TraceApi::new( +// eth_api, +// BlockingTaskGuard::new(10), +// )) +// } + +// pub(crate) fn create_eth_api( +// provider: Node::Provider, +// evm_config: Node::Evm, +// pool: Node::Pool, +// network: Node::Network, +// ) -> Arc> +// where +// Node: FullNodeComponents, +// Node::Types: NodeTypes, +// EthApi: Call + LoadPendingBlock, +// { +// let cache = EthStateCache::spawn( +// provider.clone(), +// Default::default(), +// ); + +// let fee_history_cache = FeeHistoryCache::new(FeeHistoryCacheConfig::default()); + +// let gas_oracle = GasPriceOracle::new( +// provider.clone(), +// Default::default(), +// cache.clone() +// ); + +// Arc::new(EthApi::new( +// provider.clone(), +// pool.clone(), +// network.clone(), +// cache.clone(), +// gas_oracle, +// ETHEREUM_BLOCK_GAS_LIMIT, +// DEFAULT_MAX_SIMULATE_BLOCKS, +// DEFAULT_ETH_PROOF_WINDOW, +// BlockingTaskPool::build().expect("failed to build tracing pool"), +// fee_history_cache, +// evm_config.clone(), +// DEFAULT_PROOF_PERMITS, +// )) +// } \ No newline at end of file diff --git a/crates/manager/src/common.rs b/crates/manager/src/common.rs index 7028090..41025c4 100644 --- a/crates/manager/src/common.rs +++ b/crates/manager/src/common.rs @@ -7,6 +7,8 @@ use crate::{gas::GasConfig, strategy::path_finder::PathFinderConfig, types::Cand // Strategy configuration pub const PATH_FINDER_EXEX_ID: &str = "path-finder"; +pub const INDEXER_EXEX_ID: &str = "indexer"; + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "kebab-case", tag = "type")] pub enum StrategyConfig { From 0c91b6ae0a94422c8f5d8d112d129b0c6bd008bd Mon Sep 17 00:00:00 2001 From: 0xwiederholen <0xwiederholen@gmail.com> Date: Sat, 26 Jul 2025 22:18:45 +0900 Subject: [PATCH 02/12] feat: implement rocksdb --- Cargo.lock | 85 ++++++++++++++++++- bin/searcher-reth/src/main.rs | 2 - crates/extension/src/exex.rs | 27 +++--- crates/indexer/Cargo.toml | 3 +- .../src/datasets/dolomite_borrow_position.rs | 22 ++--- crates/indexer/src/db_writer.rs | 64 ++++++++++++++ crates/indexer/src/indexer.rs | 29 ++++--- crates/indexer/src/utils.rs | 2 + rust-toolchain.toml | 3 + 9 files changed, 193 insertions(+), 44 deletions(-) create mode 100644 rust-toolchain.toml diff --git a/Cargo.lock b/Cargo.lock index 2acacca..6c187fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1430,6 +1430,26 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags 2.9.1", + "cexpr", + "clang-sys", + "itertools 0.10.5", + "lazy_static", + "lazycell", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn 2.0.104", +] + [[package]] name = "bindgen" version = "0.70.1" @@ -1448,6 +1468,24 @@ dependencies = [ "syn 2.0.104", ] +[[package]] +name = "bindgen" +version = "0.71.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" +dependencies = [ + "bitflags 2.9.1", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "proc-macro2", + "quote", + "regex", + "rustc-hash 2.1.1", + "shlex", + "syn 2.0.104", +] + [[package]] name = "bit-set" version = "0.8.0" @@ -1802,6 +1840,16 @@ dependencies = [ "serde", ] +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "c-kzg" version = "2.1.1" @@ -4491,6 +4539,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" version = "0.2.174" @@ -4550,7 +4604,7 @@ version = "0.14.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e78a09b56be5adbcad5aa1197371688dc6bb249a26da3bca2011ee2fb987ebfb" dependencies = [ - "bindgen", + "bindgen 0.70.1", "errno", "libc", ] @@ -4566,6 +4620,21 @@ dependencies = [ "redox_syscall", ] +[[package]] +name = "librocksdb-sys" +version = "0.17.1+9.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b7869a512ae9982f4d46ba482c2a304f1efd80c6412a3d4bf57bb79a619679f" +dependencies = [ + "bindgen 0.69.5", + "bzip2-sys", + "cc", + "libc", + "libz-sys", + "lz4-sys", + "zstd-sys", +] + [[package]] name = "libsecp256k1" version = "0.7.2" @@ -7351,7 +7420,7 @@ name = "reth-mdbx-sys" version = "1.5.1" source = "git+https://github.com/paradigmxyz/reth?tag=v1.5.1#dbe7ee9c21392f360ff01f6307480f5d7dd73a3a" dependencies = [ - "bindgen", + "bindgen 0.70.1", "cc", ] @@ -8972,6 +9041,16 @@ dependencies = [ "byteorder", ] +[[package]] +name = "rocksdb" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26ec73b20525cb235bad420f911473b69f9fe27cc856c5461bccd7e4af037f43" +dependencies = [ + "libc", + "librocksdb-sys", +] + [[package]] name = "rolling-file" version = "0.2.0" @@ -9414,6 +9493,7 @@ dependencies = [ "reth-revm", "reth-tracing", "reth-transaction-pool", + "rocksdb", "serde", "serde_yaml", "tokio", @@ -12031,6 +12111,7 @@ version = "2.0.15+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" dependencies = [ + "bindgen 0.71.1", "cc", "pkg-config", ] diff --git a/bin/searcher-reth/src/main.rs b/bin/searcher-reth/src/main.rs index 48fcd7b..17376b5 100644 --- a/bin/searcher-reth/src/main.rs +++ b/bin/searcher-reth/src/main.rs @@ -35,8 +35,6 @@ fn main() -> eyre::Result<()> { // Install Exex for various strategies let searcher_exex = SearcherExEx::new(wallet, signal_manager.subscribe()); - // Init Exex for Indexer - let indexer_exex = IndexerExEx::new(); let mut node_builder = builder.node(EthereumNode::default()); diff --git a/crates/extension/src/exex.rs b/crates/extension/src/exex.rs index fb830d6..fcc73f9 100644 --- a/crates/extension/src/exex.rs +++ b/crates/extension/src/exex.rs @@ -19,23 +19,20 @@ use tokio::sync::broadcast; use crate::relayer_pool::{ RelayerMessage, RelayerPool, WalletPool }; use searcher_reth_indexer::{indexer::Indexer, db_writer::DbWriter }; use alloy_consensus::{BlockHeader, Header, Block}; -use reth::primitives::{EthereumHardforks, NodePrimitives}; +use reth::primitives::{ EthereumHardforks, NodePrimitives}; use searcher_reth_indexer::utils::{Config, connect_to_postgres, create_tables}; +use searcher_reth_indexer::db_writer::RocksDB; use tokio_postgres::Client; -pub struct IndexerExEx {} +pub struct IndexerExEx; impl IndexerExEx { - pub fn new() -> Self { - Self {} - } - pub async fn init( ctx: ExExContext, ) -> Result>> where - Node::Types: NodeTypes, - ::ChainSpec: EthereumHardforks, + Node: FullNodeComponents, + //::ChainSpec: EthereumHardforks, ::Primitives: NodePrimitives< BlockHeader = Header, Block = Block, @@ -47,21 +44,24 @@ impl IndexerExEx { let client = Arc::new(connect_to_postgres().await?); create_tables(&client).await?; + + let db = RocksDB::init("rocksdb", &["default"]); // Create indexer with all processors initialized internally let indexer = Indexer::new(config); - Ok(Self::indexer_exex(ctx, client, indexer)) + Ok(Self::indexer_exex(ctx, client, db, indexer)) } async fn indexer_exex( mut ctx: ExExContext, client: Arc, + db: RocksDB, indexer: Indexer, ) -> Result<()> where Node::Types: NodeTypes, - ::ChainSpec: EthereumHardforks, + //::ChainSpec: EthereumHardforks, ::Primitives: NodePrimitives< BlockHeader = Header, Block = Block, @@ -77,10 +77,11 @@ impl IndexerExEx { if let Err(e) = indexer.process_blocks( new.blocks_and_receipts(), &client, + &db, ctx.provider().clone(), - Arc::new(ctx.evm_config().clone()), - Arc::new(ctx.pool().clone()), - Arc::new(ctx.network().clone()), + //Arc::new(ctx.evm_config().clone()), + //Arc::new(ctx.pool().clone()), + //Arc::new(ctx.network().clone()), ).await { tracing::warn!("Failed to process committed blocks: {}", e); } diff --git a/crates/indexer/Cargo.toml b/crates/indexer/Cargo.toml index 74e15a3..53702fe 100644 --- a/crates/indexer/Cargo.toml +++ b/crates/indexer/Cargo.toml @@ -45,4 +45,5 @@ futures = "0.3.31" chrono = "0.4.40" serde_yaml = "0.9.34" primitive-types = { version = "0.13.1", features = ["serde"] } -tracing = "0.1.41" \ No newline at end of file +tracing = "0.1.41" +rocksdb = "0.23.0" diff --git a/crates/indexer/src/datasets/dolomite_borrow_position.rs b/crates/indexer/src/datasets/dolomite_borrow_position.rs index 26340a9..2031f03 100644 --- a/crates/indexer/src/datasets/dolomite_borrow_position.rs +++ b/crates/indexer/src/datasets/dolomite_borrow_position.rs @@ -1,5 +1,5 @@ use crate::record_values; -use crate::db_writer::DbWriter; +use crate::db_writer::{DbWriter, RocksDB}; use alloy_sol_types::{sol, SolEvent}; use alloy_primitives::{address, Address}; use reth_primitives::{RecoveredBlock, Receipt, Block}; @@ -21,7 +21,8 @@ sol! { pub async fn process_dolomite_borrow_positions( block_data: &(RecoveredBlock, Vec), components: ProcessingComponents, - writer: &mut DbWriter + writer: &mut DbWriter, + db: &RocksDB, ) -> Result<()> { let block = &block_data.0; let receipts = &block_data.1; @@ -40,18 +41,11 @@ pub async fn process_dolomite_borrow_positions( match BorrowPositionOpen::decode_raw_log(log.topics(), &log.data.data) { Ok(create) => { - writer - .write_record(record_values![ - block_number as i64, - tx.hash(), - tx_idx as i64, - log_idx as i64, - log.address, - create._borrower, - create._borrowAccountNumber, - Utc::now(), - ]) - .await?; + db.save( + "dolomite_borrow_positions", + &format!("{}", create._borrowAccountNumber), + &format!("{}", create._borrower) + ); } Err(e) => { debug!("Failed to decode dolomite borrow position open event: {:?}", e); diff --git a/crates/indexer/src/db_writer.rs b/crates/indexer/src/db_writer.rs index 2a207bd..31bc906 100644 --- a/crates/indexer/src/db_writer.rs +++ b/crates/indexer/src/db_writer.rs @@ -5,6 +5,70 @@ use tokio_postgres::{Client, Statement, types::Type}; use futures::pin_mut; use tokio_postgres::binary_copy::BinaryCopyInWriter; use alloy_primitives::{Address, FixedBytes, Uint, Signed}; +use std::collections::{HashSet, HashMap}; +use rocksdb::{DB, ColumnFamily, Options, ColumnFamilyDescriptor}; + +#[derive(Clone)] +pub struct RocksDB { + db: Arc +} + +impl RocksDB { + pub fn init(file_path: &str, required_cfs: &[&str]) -> Self { + let existing_cfs = DB::list_cf(&Options::default(), file_path) + .unwrap_or_else(|_| vec!["default".to_string()]); + let existing_cf_set: HashSet = existing_cfs.iter().cloned().collect(); + + let cf_descriptors: Vec<_> = existing_cfs + .iter() + .map(|cf_name| ColumnFamilyDescriptor::new(cf_name, Options::default())) + .collect(); + + let mut db = DB::open_cf_descriptors(&Options::default(), file_path, cf_descriptors) + .expect("Failed to open DB"); + + for cf in required_cfs { + if !existing_cf_set.contains(*cf) { + db.create_cf(*cf, &Options::default()) + .expect(&format!("Failed to create CF: {}", cf)); + } + } + + RocksDB { + db: Arc::new(db) + } + } + + pub fn save(&self, cf: &str, k: &str, v: &str) -> bool { + let cf = self.db.cf_handle(cf).expect("missing CF"); + self.db.put_cf(cf, k.as_bytes(), v.as_bytes()).is_ok() + } + + pub fn find(&self, cf: &str, k: &str) -> Option { + let cf = self.db.cf_handle(cf).expect("missing CF"); + match self.db.get_cf(cf, k.as_bytes()) { + Ok(Some(v)) => { + let result = String::from_utf8(v).unwrap(); + println!("Finding '{}' returns '{}'", k, result); + Some(result) + }, + Ok(None) => { + println!("Finding '{}' returns None", k); + None + }, + Err(e) => { + println!("Error retrieving value for {}: {}", k, e); + None + } + } + } + + pub fn delete(&self, cf: &str, k: &str) -> bool { + let cf = self.db.cf_handle(cf).expect("missing CF"); + self.db.delete_cf(cf, k.as_bytes()).is_ok() + } +} + pub struct DbWriter { client: Arc, diff --git a/crates/indexer/src/indexer.rs b/crates/indexer/src/indexer.rs index db05fa1..52820bc 100644 --- a/crates/indexer/src/indexer.rs +++ b/crates/indexer/src/indexer.rs @@ -1,6 +1,6 @@ use crate::utils::Config; use crate::table_definitions::get_table; -use crate::db_writer::DbWriter; +use crate::db_writer::{DbWriter, RocksDB}; use crate::datasets::{ dolomite_borrow_position::process_dolomite_borrow_positions }; @@ -34,7 +34,8 @@ struct ProcessorInfo { processor: for<'a> fn( &'a (RecoveredBlock, Vec), ProcessingComponents, - &'a mut DbWriter + &'a mut DbWriter, + &'a RocksDB ) -> futures::future::BoxFuture<'a, Result<()>>, } @@ -45,7 +46,8 @@ impl ProcessorInfo { processor: for<'a> fn( &'a (RecoveredBlock, Vec), ProcessingComponents, - &'a mut DbWriter + &'a mut DbWriter, + &'a RocksDB ) -> futures::future::BoxFuture<'a, Result<()>>, ) -> Self { Self { @@ -65,7 +67,7 @@ impl Indexer where Node: FullNodeComponents + FullNodeTypes, Node::Types: NodeTypes, - ::ChainSpec: EthereumHardforks, + //::ChainSpec: EthereumHardforks, ::Primitives: NodePrimitives< BlockHeader = Header, Block = Block, @@ -98,7 +100,7 @@ where "dolomite_borrow_positions" => ProcessorInfo::new( table_name, processor_name, - |block_data, components, writer| Box::pin(process_dolomite_borrow_positions::(block_data, components, writer)) + |block_data, components, writer, db| Box::pin(process_dolomite_borrow_positions::(block_data, components, writer, db)) ), _ => return, // Skip unknown processors }; @@ -113,10 +115,11 @@ where &self, blocks_and_receipts: impl Iterator, &Vec)>, client: &Arc, + db: &RocksDB, provider: Node::Provider, - evm_config: Arc, - pool: Arc, - network: Arc, + //evm_config: Arc, + //pool: Arc, + //network: Arc, ) -> Result<()> where Node::Evm: ConfigureEvm @@ -129,7 +132,7 @@ where for (block, receipts) in blocks_and_receipts { let block_number = block.number; - let block_id = BlockId::Number(BlockNumberOrTag::from(block_number)); + //let block_id = BlockId::Number(BlockNumberOrTag::from(block_number)); // Create EthAPI // let eth_api = crate::utils::create_eth_api::( @@ -169,7 +172,7 @@ where }; let block_data = (block, receipts); - if let Err(e) = self.process_block_data(&block_data, components).await { + if let Err(e) = self.process_block_data(&block_data, components, db).await { warn!("Failed to process block {}: {}", block_number, e); } } @@ -181,10 +184,11 @@ where &self, block_data: &(RecoveredBlock, Vec), components: ProcessingComponents, + db: &RocksDB ) -> Result<()> where Node::Types: NodeTypes, - ::ChainSpec: EthereumHardforks, + //::ChainSpec: EthereumHardforks, ::Primitives: NodePrimitives< BlockHeader = Header, Block = Block, @@ -205,6 +209,7 @@ where let processor_fn = processor.processor; let block_data = block_data.clone(); let components = components.clone(); + let db = db.clone(); // Clone db before spawn let table = get_table(processor.table_name) .expect(&format!("Table definition not found for {}", processor.table_name)); @@ -215,7 +220,7 @@ where Ok(w) => w, Err(e) => return Err((processor_name, e.to_string())) }; - match processor_fn(&block_data, components, &mut writer).await { + match processor_fn(&block_data, components, &mut writer, &db).await { Ok(()) => { match writer.finish().await { Ok(records_written) => Ok((processor_name, records_written, event_start_time.elapsed())), diff --git a/crates/indexer/src/utils.rs b/crates/indexer/src/utils.rs index 4cd85e2..d75adec 100644 --- a/crates/indexer/src/utils.rs +++ b/crates/indexer/src/utils.rs @@ -15,6 +15,8 @@ use reth_tracing::tracing::info; use serde::{Deserialize, Serialize}; use std::{sync::Arc, collections::HashMap, env}; use tokio_postgres::{Client, NoTls}; +use rocksdb::{DB, Options, ColumnFamilyDescriptor, ColumnFamily, DBWithThreadMode, SingleThreaded}; +use std::collections::HashSet; #[derive(Debug, Serialize, Deserialize, Clone)] pub struct Config { diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..14eb32b --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "nightly" +components = ["rustfmt", "clippy"] \ No newline at end of file From afcf2db7e0ab96f51fea6ef97d1403e79d2c9ee8 Mon Sep 17 00:00:00 2001 From: 0xwiederholen <0xwiederholen@gmail.com> Date: Mon, 28 Jul 2025 18:05:21 +0900 Subject: [PATCH 03/12] feat: remove postgres db code --- crates/extension/src/exex.rs | 2 +- .../src/datasets/dolomite_borrow_position.rs | 6 +- crates/indexer/src/db_writer.rs | 267 ------------------ crates/indexer/src/indexer.rs | 33 +-- 4 files changed, 13 insertions(+), 295 deletions(-) diff --git a/crates/extension/src/exex.rs b/crates/extension/src/exex.rs index fcc73f9..2fd36c6 100644 --- a/crates/extension/src/exex.rs +++ b/crates/extension/src/exex.rs @@ -17,7 +17,7 @@ use searcher_reth_manager::SignalType; use searcher_reth_strategy::core::strategy::Strategy; use tokio::sync::broadcast; use crate::relayer_pool::{ RelayerMessage, RelayerPool, WalletPool }; -use searcher_reth_indexer::{indexer::Indexer, db_writer::DbWriter }; +use searcher_reth_indexer::{indexer::Indexer}; use alloy_consensus::{BlockHeader, Header, Block}; use reth::primitives::{ EthereumHardforks, NodePrimitives}; use searcher_reth_indexer::utils::{Config, connect_to_postgres, create_tables}; diff --git a/crates/indexer/src/datasets/dolomite_borrow_position.rs b/crates/indexer/src/datasets/dolomite_borrow_position.rs index 2031f03..1f119dd 100644 --- a/crates/indexer/src/datasets/dolomite_borrow_position.rs +++ b/crates/indexer/src/datasets/dolomite_borrow_position.rs @@ -1,9 +1,7 @@ -use crate::record_values; -use crate::db_writer::{DbWriter, RocksDB}; +use crate::db_writer::RocksDB; use alloy_sol_types::{sol, SolEvent}; use alloy_primitives::{address, Address}; use reth_primitives::{RecoveredBlock, Receipt, Block}; -use chrono::Utc; use eyre::Result; use reth_node_api::FullNodeComponents; use tracing::debug; @@ -21,12 +19,10 @@ sol! { pub async fn process_dolomite_borrow_positions( block_data: &(RecoveredBlock, Vec), components: ProcessingComponents, - writer: &mut DbWriter, db: &RocksDB, ) -> Result<()> { let block = &block_data.0; let receipts = &block_data.1; - let block_number = block.number; // Iterate through transactions and their receipts for (tx_idx, (tx, receipt)) in block.body().transactions.iter().zip(receipts.iter()).enumerate() { diff --git a/crates/indexer/src/db_writer.rs b/crates/indexer/src/db_writer.rs index 31bc906..5acd7d8 100644 --- a/crates/indexer/src/db_writer.rs +++ b/crates/indexer/src/db_writer.rs @@ -67,271 +67,4 @@ impl RocksDB { let cf = self.db.cf_handle(cf).expect("missing CF"); self.db.delete_cf(cf, k.as_bytes()).is_ok() } -} - - -pub struct DbWriter { - client: Arc, - revert_stmt: Statement, - table: TableDefinition, - records: Vec>>, -} - -impl DbWriter { - pub async fn new(client: &Arc, table: TableDefinition) -> Result { - let revert_stmt = client - .prepare(&table.revert_statement()) - .await?; - - Ok(Self { - client: Arc::clone(client), - revert_stmt, - table, - records: Vec::new(), - }) - } - - pub async fn write_record(&mut self, record: Vec>) -> Result<()> { - self.records.push(record); - Ok(()) - } - - pub async fn finish(self) -> Result { - if self.records.is_empty() { - return Ok(0); - } - - let column_names = self.table.columns.iter() - .map(|col| col.name) - .collect::>() - .join(", "); - - // Start COPY operation - let copy_stmt = format!( - "COPY {} ({}) FROM STDIN BINARY", - self.table.name, - column_names - ); - - let sink = self.client.copy_in(©_stmt).await?; - - // Create a binary writer with the correct types - let writer = BinaryCopyInWriter::new( - sink, - &self.table.columns.iter() - .map(|col| col.get_postgres_type()) - .collect::>() - ); - pin_mut!(writer); - - // Process all records - let mut total_records = 0; - for record in self.records { - // Convert the boxed ToSql traits to references - let record_refs: Vec<&(dyn tokio_postgres::types::ToSql + Sync)> = record - .iter() - .map(|value| value.as_ref() as &(dyn tokio_postgres::types::ToSql + Sync)) - .collect(); - - writer.as_mut().write(&record_refs).await?; - total_records += 1; - } - - // Finish the COPY operation - writer.finish().await?; - - Ok(total_records) - } - - pub async fn revert(&self, block_numbers: &[i64]) -> Result<()> { - self.client - .execute(&self.revert_stmt, &[&block_numbers]) - .await?; - Ok(()) - } -} - -// Helper trait to make it easier to convert common Ethereum data types to Postgres types -pub trait EthereumValue { - fn to_sql_value(&self) -> Box; -} - -impl EthereumValue for primitive_types::H256 { - fn to_sql_value(&self) -> Box { - Box::new(self.to_string()) - } -} - -impl EthereumValue for primitive_types::U256 { - fn to_sql_value(&self) -> Box { - Box::new(self.to_string()) - } -} - -impl EthereumValue for alloy_primitives::Address { - fn to_sql_value(&self) -> Box { - Box::new(self.to_checksum(Some(1))) - } -} - -impl EthereumValue for Option { - fn to_sql_value(&self) -> Box { - Box::new(self.clone()) - } -} - -impl<'a> EthereumValue for Option<&'a str> { - fn to_sql_value(&self) -> Box { - Box::new(self.map(|s| s.to_string())) - } -} - -impl EthereumValue for Option { - fn to_sql_value(&self) -> Box { - Box::new(self.clone()) - } -} - -impl EthereumValue for Option { - fn to_sql_value(&self) -> Box { - Box::new(self.map(|v| v as i64)) - } -} - -impl EthereumValue for Option { - fn to_sql_value(&self) -> Box { - Box::new(self.clone()) - } -} - -impl EthereumValue for Option> { - fn to_sql_value(&self) -> Box { - Box::new(self.clone()) - } -} - -impl EthereumValue for Option { - fn to_sql_value(&self) -> Box { - Box::new(self.as_ref().map(|addr| addr.to_checksum(Some(1)))) - } -} - -impl EthereumValue for Option { - fn to_sql_value(&self) -> Box { - Box::new(self.as_ref().map(|h| h.to_string())) - } -} - -impl EthereumValue for Option { - fn to_sql_value(&self) -> Box { - Box::new(self.as_ref().map(|u| u.to_string())) - } -} - -impl EthereumValue for Option> { - fn to_sql_value(&self) -> Box { - Box::new(self.as_ref().map(|bytes| bytes.to_string())) - } -} - -impl<'a> EthereumValue for Option<&'a alloy_primitives::FixedBytes<32>> { - fn to_sql_value(&self) -> Box { - Box::new(self.map(|bytes| bytes.to_string())) - } -} - -impl EthereumValue for Option> { - fn to_sql_value(&self) -> Box { - Box::new(self.as_ref().map(|u| u.to_string())) - } -} - -impl EthereumValue for Vec { - fn to_sql_value(&self) -> Box { - Box::new(alloy_primitives::hex::encode_prefixed(self)) - } -} - -impl EthereumValue for [u8] { - fn to_sql_value(&self) -> Box { - Box::new(alloy_primitives::hex::encode_prefixed(self)) - } -} - -impl EthereumValue for String { - fn to_sql_value(&self) -> Box { - Box::new(self.clone()) - } -} - -impl EthereumValue for str { - fn to_sql_value(&self) -> Box { - Box::new(self.to_string()) - } -} - -impl EthereumValue for i64 { - fn to_sql_value(&self) -> Box { - Box::new(*self) - } -} - -impl EthereumValue for i32 { - fn to_sql_value(&self) -> Box { - Box::new(*self as i64) - } -} - -impl EthereumValue for bool { - fn to_sql_value(&self) -> Box { - Box::new(*self) - } -} - -impl EthereumValue for chrono::DateTime { - fn to_sql_value(&self) -> Box { - Box::new(*self) - } -} - -impl EthereumValue for alloy_primitives::FixedBytes<32> { - fn to_sql_value(&self) -> Box { - Box::new(self.to_string()) - } -} - -impl EthereumValue for alloy_primitives::Uint { - fn to_sql_value(&self) -> Box { - Box::new(self.to_string()) - } -} - -impl EthereumValue for alloy_primitives::Signed { - fn to_sql_value(&self) -> Box { - Box::new(self.to_string()) - } -} - -// Implement for references to types that implement EthereumValue -impl EthereumValue for &T { - fn to_sql_value(&self) -> Box { - (*self).to_sql_value() - } -} - -impl EthereumValue for f64 { - fn to_sql_value(&self) -> Box { - Box::new(*self) - } -} - -// Macro to help convert datatypes to Postgres types (via EthereumValue trait implementations) -#[macro_export] -macro_rules! record_values { - ($($value:expr),* $(,)?) => {{ - // Import here so it does not need to be imported in dataset files - use $crate::db_writer::EthereumValue; - let values: Vec> = vec![$($value.to_sql_value()),*]; - values - }}; } \ No newline at end of file diff --git a/crates/indexer/src/indexer.rs b/crates/indexer/src/indexer.rs index 52820bc..02a4a38 100644 --- a/crates/indexer/src/indexer.rs +++ b/crates/indexer/src/indexer.rs @@ -1,6 +1,6 @@ use crate::utils::Config; use crate::table_definitions::get_table; -use crate::db_writer::{DbWriter, RocksDB}; +use crate::db_writer::RocksDB; use crate::datasets::{ dolomite_borrow_position::process_dolomite_borrow_positions }; @@ -34,7 +34,6 @@ struct ProcessorInfo { processor: for<'a> fn( &'a (RecoveredBlock, Vec), ProcessingComponents, - &'a mut DbWriter, &'a RocksDB ) -> futures::future::BoxFuture<'a, Result<()>>, } @@ -46,7 +45,6 @@ impl ProcessorInfo { processor: for<'a> fn( &'a (RecoveredBlock, Vec), ProcessingComponents, - &'a mut DbWriter, &'a RocksDB ) -> futures::future::BoxFuture<'a, Result<()>>, ) -> Self { @@ -100,7 +98,7 @@ where "dolomite_borrow_positions" => ProcessorInfo::new( table_name, processor_name, - |block_data, components, writer, db| Box::pin(process_dolomite_borrow_positions::(block_data, components, writer, db)) + |block_data, components, db| Box::pin(process_dolomite_borrow_positions::(block_data, components, db)) ), _ => return, // Skip unknown processors }; @@ -210,23 +208,14 @@ where let block_data = block_data.clone(); let components = components.clone(); let db = db.clone(); // Clone db before spawn - let table = get_table(processor.table_name) - .expect(&format!("Table definition not found for {}", processor.table_name)); + // let table = get_table(processor.table_name) + // .expect(&format!("Table definition not found for {}", processor.table_name)); // Spawn the task let task = tokio::spawn(async move { let event_start_time = Instant::now(); - let mut writer = match DbWriter::new(&components.client, table).await { - Ok(w) => w, - Err(e) => return Err((processor_name, e.to_string())) - }; - match processor_fn(&block_data, components, &mut writer, &db).await { - Ok(()) => { - match writer.finish().await { - Ok(records_written) => Ok((processor_name, records_written, event_start_time.elapsed())), - Err(e) => Err((processor_name, e.to_string())) - } - }, + match processor_fn(&block_data, components, &db).await { + Ok(()) => Ok((processor_name, event_start_time.elapsed())), Err(e) => Err((processor_name, e.to_string())) } }); @@ -242,9 +231,9 @@ where for task in tasks { match task.await { - Ok(Ok((name, records, duration))) => { - total_records += records; - event_results.push((name, records, duration)); + Ok(Ok((name, duration))) => { + total_records += 1; // Assuming each event writes one record for now + event_results.push((name, duration)); } Ok(Err((name, error))) => { failed_events.push((name, error)); @@ -262,8 +251,8 @@ where if !event_results.is_empty() { let events_summary: Vec = event_results .iter() - .map(|(name, records, time)| { - format!("{}({}, {:.2}s)", name, records, time.as_secs_f64()) + .map(|(name, time)| { + format!("{}({}, {:.2}s)", name, 1, time.as_secs_f64()) }) .collect(); From 057205c32c8d79e7db611bd382388305fcf2cf81 Mon Sep 17 00:00:00 2001 From: 0xwiederholen <0xwiederholen@gmail.com> Date: Sat, 2 Aug 2025 00:16:47 +0900 Subject: [PATCH 04/12] feat: add aave protocol borrow event --- .../src/datasets/aave_execute_borrow.rs | 60 +++++++++++++++++++ .../src/datasets/dolomite_borrow_position.rs | 4 +- crates/indexer/src/datasets/mod.rs | 3 +- crates/indexer/src/indexer.rs | 9 ++- 4 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 crates/indexer/src/datasets/aave_execute_borrow.rs diff --git a/crates/indexer/src/datasets/aave_execute_borrow.rs b/crates/indexer/src/datasets/aave_execute_borrow.rs new file mode 100644 index 0000000..089ae37 --- /dev/null +++ b/crates/indexer/src/datasets/aave_execute_borrow.rs @@ -0,0 +1,60 @@ +use crate::db_writer::RocksDB; +use alloy_sol_types::{sol, SolEvent}; +use alloy_primitives::{address, Address}; +use reth_primitives::{RecoveredBlock, Receipt, Block}; +use eyre::Result; +use reth_node_api::FullNodeComponents; +use tracing::debug; +use crate::indexer::ProcessingComponents; +// Pool contarct address +const ARBITRUM_AAVE_CONTRACT_ADDRESS: Address = address!("0x794a61358D6845594F94dc1DB02A252b5b4814aD"); + +sol! { + event Borrow( + address indexed reserve, + address user, + address indexed onBehalfOf, + uint256 amount, + uint8 interestRateMode, + uint256 borrowRate, + uint16 indexed referralCode + ); +} + +pub async fn process_aave_execute_borrow( + block_data: &(RecoveredBlock, Vec), + components: ProcessingComponents, + db: &RocksDB, +) -> Result<()> { + let block = &block_data.0; + let receipts = &block_data.1; + + // Iterate through transactions and their receipts + for (tx_idx, (tx, receipt)) in block.body().transactions.iter().zip(receipts.iter()).enumerate() { + // Process each log in the receipt + for (log_idx, log) in receipt.logs.iter().enumerate() { + // Filter on dolomite contract address + if log.address == ARBITRUM_AAVE_CONTRACT_ADDRESS { + // First check if this log matches our event signature + if log.topics().get(0) != Some(&Borrow::SIGNATURE_HASH) { + continue; + } + + match Borrow::decode_raw_log(log.topics(), &log.data.data) { + Ok(create) => { + db.save( + "aave_borrow", + &format!("{}", create.onBehalfOf), + &format!("{}_{}", create.reserve, create.amount) + ); + } + Err(e) => { + debug!("Failed to decode aave borrow event: {:?}", e); + } + } + } + } + } + Ok(()) +} + diff --git a/crates/indexer/src/datasets/dolomite_borrow_position.rs b/crates/indexer/src/datasets/dolomite_borrow_position.rs index 1f119dd..e3bda17 100644 --- a/crates/indexer/src/datasets/dolomite_borrow_position.rs +++ b/crates/indexer/src/datasets/dolomite_borrow_position.rs @@ -7,7 +7,7 @@ use reth_node_api::FullNodeComponents; use tracing::debug; use crate::indexer::ProcessingComponents; // BorrowPositionProxyV2 address -const DOLOMITE_CONTRACT_ADDRESS: Address = address!("0xC06271eb97d960F4034DDF953e16271CcB2B10BD"); +const BERA_DOLOMITE_CONTRACT_ADDRESS: Address = address!("0xC06271eb97d960F4034DDF953e16271CcB2B10BD"); sol! { event BorrowPositionOpen( @@ -29,7 +29,7 @@ pub async fn process_dolomite_borrow_positions( // Process each log in the receipt for (log_idx, log) in receipt.logs.iter().enumerate() { // Filter on dolomite contract address - if log.address == DOLOMITE_CONTRACT_ADDRESS { + if log.address == BERA_DOLOMITE_CONTRACT_ADDRESS { // First check if this log matches our event signature if log.topics().get(0) != Some(&BorrowPositionOpen::SIGNATURE_HASH) { continue; diff --git a/crates/indexer/src/datasets/mod.rs b/crates/indexer/src/datasets/mod.rs index c1c25cd..6b72da2 100644 --- a/crates/indexer/src/datasets/mod.rs +++ b/crates/indexer/src/datasets/mod.rs @@ -1 +1,2 @@ -pub mod dolomite_borrow_position; \ No newline at end of file +pub mod dolomite_borrow_position; +pub mod aave_execute_borrow; \ No newline at end of file diff --git a/crates/indexer/src/indexer.rs b/crates/indexer/src/indexer.rs index 02a4a38..9cc21d0 100644 --- a/crates/indexer/src/indexer.rs +++ b/crates/indexer/src/indexer.rs @@ -2,7 +2,8 @@ use crate::utils::Config; use crate::table_definitions::get_table; use crate::db_writer::RocksDB; use crate::datasets::{ - dolomite_borrow_position::process_dolomite_borrow_positions + dolomite_borrow_position::process_dolomite_borrow_positions, + aave_execute_borrow::process_aave_execute_borrow, }; use alloy_consensus::{Header}; use alloy_rpc_types::{BlockId, BlockNumberOrTag}; @@ -88,6 +89,7 @@ where // Register all available processors // e.g) indexer.add_processor("headers", "Headers"); indexer.add_processor("dolomite_borrow_positions", "DolomiteBorrowPositions"); + //indexer.add_processor("aave_borrow", "AaveBorrow"); info!("Initialized indexer with processors: {:?}", indexer.list_processors()); indexer @@ -100,6 +102,11 @@ where processor_name, |block_data, components, db| Box::pin(process_dolomite_borrow_positions::(block_data, components, db)) ), + // "aave_borrow" => ProcessorInfo::new( + // table_name, + // processor_name, + // |block_data, components, db| Box::pin(process_aave_execute_borrow::(block_data, components, db)) + // ), _ => return, // Skip unknown processors }; self.processors.push(processor); From 8a42efbb932aa96c92108b1c0bcd8e8db78a988e Mon Sep 17 00:00:00 2001 From: 0xwiederholen <0xwiederholen@gmail.com> Date: Tue, 12 Aug 2025 07:11:34 +0900 Subject: [PATCH 05/12] feat: remove postgres --- Cargo.lock | 123 --------------------- crates/extension/Cargo.toml | 2 +- crates/extension/src/exex.rs | 13 ++- crates/indexer/Cargo.toml | 2 +- crates/indexer/src/db_writer.rs | 3 - crates/indexer/src/indexer.rs | 11 +- crates/indexer/src/lib.rs | 1 - crates/indexer/src/table_definitions.rs | 135 ------------------------ crates/indexer/src/utils.rs | 42 -------- 9 files changed, 11 insertions(+), 321 deletions(-) delete mode 100644 crates/indexer/src/table_definitions.rs diff --git a/Cargo.lock b/Cargo.lock index 6c187fe..275e5d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2969,12 +2969,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "fallible-iterator" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" - [[package]] name = "fast-float2" version = "0.2.3" @@ -4841,16 +4835,6 @@ dependencies = [ "regex-automata 0.1.10", ] -[[package]] -name = "md-5" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" -dependencies = [ - "cfg-if", - "digest 0.10.7", -] - [[package]] name = "memchr" version = "2.7.5" @@ -5615,36 +5599,6 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" -[[package]] -name = "postgres-protocol" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76ff0abab4a9b844b93ef7b81f1efc0a366062aaef2cd702c76256b5dc075c54" -dependencies = [ - "base64 0.22.1", - "byteorder", - "bytes", - "fallible-iterator", - "hmac", - "md-5", - "memchr", - "rand 0.9.1", - "sha2 0.10.9", - "stringprep", -] - -[[package]] -name = "postgres-types" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613283563cd90e1dfc3518d548caee47e0e725455ed619881f5cf21f36de4b48" -dependencies = [ - "bytes", - "chrono", - "fallible-iterator", - "postgres-protocol", -] - [[package]] name = "potential_utf" version = "0.1.2" @@ -9458,7 +9412,6 @@ dependencies = [ "searcher-reth-manager", "searcher-reth-strategy", "tokio", - "tokio-postgres", ] [[package]] @@ -9497,7 +9450,6 @@ dependencies = [ "serde", "serde_yaml", "tokio", - "tokio-postgres", "tracing", ] @@ -10036,17 +9988,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" -[[package]] -name = "stringprep" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" -dependencies = [ - "unicode-bidi", - "unicode-normalization", - "unicode-properties", -] - [[package]] name = "strsim" version = "0.11.1" @@ -10437,32 +10378,6 @@ dependencies = [ "syn 2.0.104", ] -[[package]] -name = "tokio-postgres" -version = "0.7.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c95d533c83082bb6490e0189acaa0bbeef9084e60471b696ca6988cd0541fb0" -dependencies = [ - "async-trait", - "byteorder", - "bytes", - "fallible-iterator", - "futures-channel", - "futures-util", - "log", - "parking_lot", - "percent-encoding", - "phf", - "pin-project-lite", - "postgres-protocol", - "postgres-types", - "rand 0.9.1", - "socket2", - "tokio", - "tokio-util", - "whoami", -] - [[package]] name = "tokio-rustls" version = "0.24.1" @@ -10865,33 +10780,12 @@ version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" -[[package]] -name = "unicode-bidi" -version = "0.3.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" - [[package]] name = "unicode-ident" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" -[[package]] -name = "unicode-normalization" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" -dependencies = [ - "tinyvec", -] - -[[package]] -name = "unicode-properties" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" - [[package]] name = "unicode-segmentation" version = "1.12.0" @@ -11104,12 +10998,6 @@ dependencies = [ "wit-bindgen-rt", ] -[[package]] -name = "wasite" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" - [[package]] name = "wasm-bindgen" version = "0.2.100" @@ -11270,17 +11158,6 @@ dependencies = [ "rustls-pki-types", ] -[[package]] -name = "whoami" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6994d13118ab492c3c80c1f81928718159254c53c472bf9ce36f8dae4add02a7" -dependencies = [ - "redox_syscall", - "wasite", - "web-sys", -] - [[package]] name = "widestring" version = "1.2.0" diff --git a/crates/extension/Cargo.toml b/crates/extension/Cargo.toml index dbc31db..c956453 100644 --- a/crates/extension/Cargo.toml +++ b/crates/extension/Cargo.toml @@ -48,4 +48,4 @@ rayon.workspace = true reth-transaction-pool.workspace = true # postgres -tokio-postgres = { version = "0.7.13", features = ["with-chrono-0_4"] } \ No newline at end of file +#tokio-postgres = { version = "0.7.13", features = ["with-chrono-0_4"] } \ No newline at end of file diff --git a/crates/extension/src/exex.rs b/crates/extension/src/exex.rs index 2fd36c6..64b7b4b 100644 --- a/crates/extension/src/exex.rs +++ b/crates/extension/src/exex.rs @@ -20,9 +20,8 @@ use crate::relayer_pool::{ RelayerMessage, RelayerPool, WalletPool }; use searcher_reth_indexer::{indexer::Indexer}; use alloy_consensus::{BlockHeader, Header, Block}; use reth::primitives::{ EthereumHardforks, NodePrimitives}; -use searcher_reth_indexer::utils::{Config, connect_to_postgres, create_tables}; +use searcher_reth_indexer::utils::Config; use searcher_reth_indexer::db_writer::RocksDB; -use tokio_postgres::Client; pub struct IndexerExEx; @@ -42,20 +41,20 @@ impl IndexerExEx { { let config = Config::load().wrap_err("Failed to load configuration")?; - let client = Arc::new(connect_to_postgres().await?); - create_tables(&client).await?; + //let client = Arc::new(connect_to_postgres().await?); + //create_tables(&client).await?; let db = RocksDB::init("rocksdb", &["default"]); // Create indexer with all processors initialized internally let indexer = Indexer::new(config); - Ok(Self::indexer_exex(ctx, client, db, indexer)) + Ok(Self::indexer_exex(ctx, db, indexer)) } async fn indexer_exex( mut ctx: ExExContext, - client: Arc, + //client: Arc, db: RocksDB, indexer: Indexer, ) -> Result<()> @@ -76,7 +75,7 @@ impl IndexerExEx { // Process the committed blocks if let Err(e) = indexer.process_blocks( new.blocks_and_receipts(), - &client, + //&client, &db, ctx.provider().clone(), //Arc::new(ctx.evm_config().clone()), diff --git a/crates/indexer/Cargo.toml b/crates/indexer/Cargo.toml index 53702fe..a76762d 100644 --- a/crates/indexer/Cargo.toml +++ b/crates/indexer/Cargo.toml @@ -37,7 +37,7 @@ tokio.workspace = true reth-transaction-pool.workspace = true -tokio-postgres = { version = "0.7.13", features = ["with-chrono-0_4"] } +#tokio-postgres = { version = "0.7.13", features = ["with-chrono-0_4"] } serde = { workspace = true, features = ["derive"] } lazy_static = "1.5.0" eyre.workspace = true diff --git a/crates/indexer/src/db_writer.rs b/crates/indexer/src/db_writer.rs index 5acd7d8..9ab033e 100644 --- a/crates/indexer/src/db_writer.rs +++ b/crates/indexer/src/db_writer.rs @@ -1,9 +1,6 @@ -use crate::table_definitions::TableDefinition; use eyre::Result; use std::sync::Arc; -use tokio_postgres::{Client, Statement, types::Type}; use futures::pin_mut; -use tokio_postgres::binary_copy::BinaryCopyInWriter; use alloy_primitives::{Address, FixedBytes, Uint, Signed}; use std::collections::{HashSet, HashMap}; use rocksdb::{DB, ColumnFamily, Options, ColumnFamilyDescriptor}; diff --git a/crates/indexer/src/indexer.rs b/crates/indexer/src/indexer.rs index 9cc21d0..3ae1346 100644 --- a/crates/indexer/src/indexer.rs +++ b/crates/indexer/src/indexer.rs @@ -1,5 +1,4 @@ use crate::utils::Config; -use crate::table_definitions::get_table; use crate::db_writer::RocksDB; use crate::datasets::{ dolomite_borrow_position::process_dolomite_borrow_positions, @@ -7,17 +6,13 @@ use crate::datasets::{ }; use alloy_consensus::{Header}; use alloy_rpc_types::{BlockId, BlockNumberOrTag}; -// use alloy_rpc_types_trace::parity::{TraceResultsWithTransactionHash, TraceType}; use eyre::Result; use reth::builder::NodeTypes; use reth::primitives::{EthereumHardforks, NodePrimitives}; use reth_primitives::{TransactionSigned, Receipt, RecoveredBlock, Block}; use reth_node_api::{ConfigureEvm, FullNodeComponents, FullNodeTypes}; -// use reth_rpc::EthApi; -// use reth_rpc_eth_api::{FullEthApiTypes, helpers::{Call, LoadPendingBlock}}; use reth_tracing::tracing::{info, warn}; use std::{sync::Arc, time::Instant, collections::HashSet}; -use tokio_postgres::Client; // Structure to hold all the components needed for processing #[derive(Clone)] @@ -25,7 +20,7 @@ pub struct ProcessingComponents { // pub eth_api: Arc>, // pub block_traces: Option>, pub provider: Node::Provider, - pub client: Arc, + //pub client: Arc, pub config: Config, } @@ -119,7 +114,7 @@ where pub async fn process_blocks( &self, blocks_and_receipts: impl Iterator, &Vec)>, - client: &Arc, + //client: &Arc, db: &RocksDB, provider: Node::Provider, //evm_config: Arc, @@ -172,7 +167,7 @@ where // eth_api: eth_api.clone(), // block_traces, provider: provider.clone(), - client: Arc::clone(client), + //client: Arc::clone(client), config: self.config.clone(), }; diff --git a/crates/indexer/src/lib.rs b/crates/indexer/src/lib.rs index a3bdc97..9f1952e 100644 --- a/crates/indexer/src/lib.rs +++ b/crates/indexer/src/lib.rs @@ -1,5 +1,4 @@ pub mod indexer; pub mod db_writer; -pub mod table_definitions; pub mod utils; pub mod datasets; \ No newline at end of file diff --git a/crates/indexer/src/table_definitions.rs b/crates/indexer/src/table_definitions.rs deleted file mode 100644 index cbda165..0000000 --- a/crates/indexer/src/table_definitions.rs +++ /dev/null @@ -1,135 +0,0 @@ -use lazy_static::lazy_static; -use tokio_postgres::types::Type; - -#[derive(Debug, Clone)] -pub struct TableDefinition { - pub name: &'static str, - pub columns: Vec, - pub indexes: Vec<&'static str>, -} - -#[derive(Debug, Clone)] -pub struct ColumnDefinition { - pub name: &'static str, - pub sql_type: &'static str, - pub nullable: bool, - pub primary_key: bool, -} - -impl ColumnDefinition { - pub fn get_postgres_type(&self) -> Type { - match self.sql_type { - "BIGINT" => Type::INT8, - "INTEGER" => Type::INT4, - "SMALLINT" => Type::INT2, - "TEXT" | "VARCHAR" => Type::TEXT, - "BOOLEAN" => Type::BOOL, - "DOUBLE PRECISION" => Type::FLOAT8, - "REAL" => Type::FLOAT4, - "TIMESTAMP WITH TIME ZONE" => Type::TIMESTAMPTZ, - "TIMESTAMP" => Type::TIMESTAMP, - "DATE" => Type::DATE, - "BYTEA" => Type::BYTEA, - "JSON" => Type::JSON, - "JSONB" => Type::JSONB, - _ => Type::TEXT, // Default to TEXT for unknown types - } - } -} - -impl TableDefinition { - pub fn create_table_sql(&self) -> String { - let columns: Vec = self.columns - .iter() - .map(|col| { - let nullable = if col.nullable { "" } else { " NOT NULL" }; - let pk = if col.primary_key { " PRIMARY KEY" } else { "" }; - format!("{} {}{}{}", col.name, col.sql_type, nullable, pk) - }) - .collect(); - - format!( - "CREATE TABLE IF NOT EXISTS {} (\n {}\n)", - self.name, - columns.join(",\n ") - ) - } - - pub fn create_index_statements(&self) -> Vec { - self.indexes.iter().map(|idx| idx.to_string()).collect() - } - - pub fn revert_statement(&self) -> String { - format!("DELETE FROM {} WHERE block_number = ANY($1::bigint[])", self.name) - } -} - -// Define all table schemas -pub fn get_table_definitions() -> Vec { - vec![ - TableDefinition { - name: "dolomite_borrow_positions", - columns: vec![ - ColumnDefinition { - name: "block_number", - sql_type: "BIGINT", - nullable: false, - primary_key: false, - }, - ColumnDefinition { - name: "transaction_hash", - sql_type: "VARCHAR", - nullable: false, - primary_key: false, - }, - ColumnDefinition { - name: "transaction_index", - sql_type: "BIGINT", - nullable: false, - primary_key: false, - }, - ColumnDefinition { - name: "log_index", - sql_type: "BIGINT", - nullable: false, - primary_key: false, - }, - ColumnDefinition { - name: "log_address", - sql_type: "VARCHAR", - nullable: false, - primary_key: false, - }, - ColumnDefinition { - name: "borrower", - sql_type: "VARCHAR", - nullable: false, - primary_key: false, - }, - ColumnDefinition { - name: "borrower_number", - sql_type: "BIGINT", - nullable: false, - primary_key: false, - }, - ], - indexes: vec![ - "CREATE INDEX IF NOT EXISTS idx_dolomite_borrow_positions_borrower ON dolomite_borrow_positions (borrower)", - ], - }, - ] -} - -// Lazy static reference to table definitions for easy access -lazy_static! { - pub static ref TABLES: Vec = get_table_definitions(); -} - -// Helper function to get a specific table definition -pub fn get_table_definition(name: &str) -> Option { - TABLES.iter().find(|t| t.name == name).cloned() -} - -pub fn get_table(name: &str) -> Option { - get_table_definition(name) -} \ No newline at end of file diff --git a/crates/indexer/src/utils.rs b/crates/indexer/src/utils.rs index d75adec..68e0e43 100644 --- a/crates/indexer/src/utils.rs +++ b/crates/indexer/src/utils.rs @@ -1,4 +1,3 @@ -use crate::table_definitions::TABLES; // use alloy_eips::eip1559::ETHEREUM_BLOCK_GAS_LIMIT; // use reth::builder::NodeTypes; // use reth_node_api::FullNodeComponents; @@ -14,7 +13,6 @@ use crate::table_definitions::TABLES; use reth_tracing::tracing::info; use serde::{Deserialize, Serialize}; use std::{sync::Arc, collections::HashMap, env}; -use tokio_postgres::{Client, NoTls}; use rocksdb::{DB, Options, ColumnFamilyDescriptor, ColumnFamily, DBWithThreadMode, SingleThreaded}; use std::collections::HashSet; @@ -45,46 +43,6 @@ impl Config { } } -/// Sanitize a byte slice to ensure it's valid UTF-8. -pub(crate) fn sanitize_bytes(input: &[u8]) -> String { - // First, remove all null bytes - let without_nulls: Vec = input.iter().filter(|&&b| b != 0).cloned().collect(); - - // Then, convert to a string, replacing any invalid UTF-8 sequences - String::from_utf8_lossy(&without_nulls).into_owned() -} - -/// Connect to the Postgres database. -pub async fn connect_to_postgres() -> eyre::Result { - let db_url = env::var("DATABASE_URL").expect("DATABASE_URL must be set"); - let (client, connection) = tokio_postgres::connect(&db_url, NoTls).await?; - - tokio::spawn(async move { - if let Err(e) = connection.await { - eprintln!("Connection error: {}", e); - } - }); - - Ok(client) -} - -/// Create Postgres tables if they do not exist. -pub async fn create_tables(client: &Client) -> eyre::Result<()> { - for table in TABLES.iter() { - // Create the table - let create_table_sql = table.create_table_sql(); - client.execute(&create_table_sql, &[]).await?; - - // Create the indexes - for index_sql in table.create_index_statements() { - client.execute(&index_sql, &[]).await?; - } - } - - info!("Initialized database tables"); - Ok(()) -} - // Create a trace API instance with all the required components and trait bounds // pub fn create_trace_api( // provider: Node::Provider, From 85d57f1b46c5fa3f1750da50d0741d0968a8a0ba Mon Sep 17 00:00:00 2001 From: 0xwiederholen <0xwiederholen@gmail.com> Date: Tue, 12 Aug 2025 08:45:39 +0900 Subject: [PATCH 06/12] feat: impl strategyConfig for LiquidotorConfig --- bin/searcher-reth/src/main.rs | 4 +- crates/manager/Cargo.toml | 2 + crates/manager/src/common.rs | 61 +++++++++++++++++-- .../src/strategy/liquidator/config.yaml | 0 crates/manager/src/strategy/liquidator/mod.rs | 42 +++++++++++++ crates/manager/src/strategy/mod.rs | 1 + 6 files changed, 102 insertions(+), 8 deletions(-) create mode 100644 crates/manager/src/strategy/liquidator/config.yaml create mode 100644 crates/manager/src/strategy/liquidator/mod.rs diff --git a/bin/searcher-reth/src/main.rs b/bin/searcher-reth/src/main.rs index 17376b5..904441e 100644 --- a/bin/searcher-reth/src/main.rs +++ b/bin/searcher-reth/src/main.rs @@ -12,7 +12,7 @@ use searcher_reth_extension::{ }, util::signal_manager::SignalManager, }; -use searcher_reth_manager::{common::{INDEXER_EXEX_ID, PATH_FINDER_EXEX_ID}, manager::ConfigManager}; +use searcher_reth_manager::{common::{LIQUIDATOR_EXEX_ID, PATH_FINDER_EXEX_ID}, manager::ConfigManager}; fn main() -> eyre::Result<()> { let config = Arc::new(RwLock::new(ConfigManager::from_file("env.toml")?)); @@ -47,7 +47,7 @@ fn main() -> eyre::Result<()> { }); // Install Indexer - node_builder = node_builder.install_exex(INDEXER_EXEX_ID, { + node_builder = node_builder.install_exex(LIQUIDATOR_EXEX_ID, { IndexerExEx::init }); diff --git a/crates/manager/Cargo.toml b/crates/manager/Cargo.toml index f215a3b..878c296 100644 --- a/crates/manager/Cargo.toml +++ b/crates/manager/Cargo.toml @@ -17,3 +17,5 @@ alloy-primitives.workspace = true reth-revm.workspace = true serde_json.workspace = true alloy-serde = "1.0.20" +rocksdb.workspace = true +serde_yaml.workspace = true diff --git a/crates/manager/src/common.rs b/crates/manager/src/common.rs index 41025c4..7a13918 100644 --- a/crates/manager/src/common.rs +++ b/crates/manager/src/common.rs @@ -3,18 +3,19 @@ use reth_revm::state::Bytecode; use reth_tracing::tracing; use serde::{Deserialize, Serialize}; -use crate::{gas::GasConfig, strategy::path_finder::PathFinderConfig, types::CandidateEntry}; +use crate::{gas::GasConfig, strategy::{path_finder::PathFinderConfig, liquidator::LiquidatorConfig}, types::CandidateEntry}; // Strategy configuration pub const PATH_FINDER_EXEX_ID: &str = "path-finder"; -pub const INDEXER_EXEX_ID: &str = "indexer"; +pub const LIQUIDATOR_EXEX_ID: &str = "liquidator"; #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "kebab-case", tag = "type")] pub enum StrategyConfig { #[serde(rename = "path-finder")] PathFinder(PathFinderConfig), - // #[serde(rename = "liquidation")] Liquidation(LiquidationConfig), + #[serde(rename = "liquidator")] + Liquidator(LiquidatorConfig), // #[serde(rename = "arbitrage")] Arbitrage(ArbitrageConfig), } @@ -34,6 +35,7 @@ impl CommonStrategyConfig for StrategyConfig { fn get_exex_id(&self) -> &'static str { match self { StrategyConfig::PathFinder(_) => PATH_FINDER_EXEX_ID, + StrategyConfig::Liquidator(_) => LIQUIDATOR_EXEX_ID, // TODO: Add other strategy configurations } } @@ -44,6 +46,7 @@ impl CommonStrategyConfig for StrategyConfig { U256::from(config.min_liquidity.parse::().unwrap() * ONE_ETHER), U256::from(config.max_liquidity.parse::().unwrap() * ONE_ETHER), ), + StrategyConfig::Liquidator(_) => (U256::ZERO, U256::ZERO), } } @@ -58,7 +61,18 @@ impl CommonStrategyConfig for StrategyConfig { tracing::warn!("Invalid vault address, using default ZERO address"); Address::ZERO } - }, // TODO: Add other strategy configurations + }, + StrategyConfig::Liquidator(config) => match config.vault.parse() { + Ok(address) => { + tracing::info!("Using vault address: {:?}", address); + address + } + Err(_) => { + tracing::warn!("Invalid vault address, using default ZERO address"); + Address::ZERO + } + } + // TODO: Add other strategy configurations } } @@ -83,7 +97,28 @@ impl CommonStrategyConfig for StrategyConfig { Bytecode::default() // Return an empty bytecode on error } } - } // TODO: Add other strategy configurations + }, + StrategyConfig::Liquidator(config) => { + tracing::info!("Loading strategy contract for Liquidator: {:?}", config.contract); + let bytes = match hex::decode(config.contract.clone()) { + Ok(vec) => alloy_primitives::Bytes::from(vec), + Err(e) => { + tracing::error!("Failed to decode contract hex: {}", e); + return Bytecode::default(); + } + }; + match Bytecode::new_raw_checked(bytes) { + Ok(code) => { + tracing::info!("Loaded strategy contract: {:?}", code); + code + } + Err(e) => { + tracing::error!("Failed to load strategy contract: {}", e); + Bytecode::default() // Return an empty bytecode on error + } + } + } + // TODO: Add other strategy configurations } } @@ -96,13 +131,23 @@ impl CommonStrategyConfig for StrategyConfig { U256::from((max_profit * (ONE_ETHER as f64)) as u128), U256::from((min_profit * (ONE_ETHER as f64)) as u128), ) - } // TODO: Add other strategy configurations + }, + StrategyConfig::Liquidator(config) => { + let max_profit = config.max_profit.parse::().unwrap(); + let min_profit = config.min_profit.parse::().unwrap(); + ( + U256::from((max_profit * (ONE_ETHER as f64)) as u128), + U256::from((min_profit * (ONE_ETHER as f64)) as u128), + ) + } + // TODO: Add other strategy configurations } } fn get_gas_config(&self) -> GasConfig { match self { StrategyConfig::PathFinder(config) => config.gas_config.clone(), + StrategyConfig::Liquidator(config) => config.gas_config.clone(), } } @@ -123,6 +168,10 @@ impl CommonStrategyConfig for StrategyConfig { vec![] // Return an empty vector on error } } + }, + StrategyConfig::Liquidator(config) => { + // TODO: what candidates to load for liquidator? + vec![] } } } diff --git a/crates/manager/src/strategy/liquidator/config.yaml b/crates/manager/src/strategy/liquidator/config.yaml new file mode 100644 index 0000000..e69de29 diff --git a/crates/manager/src/strategy/liquidator/mod.rs b/crates/manager/src/strategy/liquidator/mod.rs new file mode 100644 index 0000000..05384f1 --- /dev/null +++ b/crates/manager/src/strategy/liquidator/mod.rs @@ -0,0 +1,42 @@ +use serde::{Deserialize, Serialize}; +use std::{collections::HashMap}; +use rocksdb::{DB, Options, ColumnFamilyDescriptor, ColumnFamily, DBWithThreadMode, SingleThreaded}; +use crate::gas::GasConfig; + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct LiquidatorConfig { + pub vault: String, + pub contract: String, + + // gas configuration + pub gas_config: GasConfig, + + pub max_profit: String, + pub min_profit: String, + + + pub enabled_events: HashMap, + #[serde(flatten)] + pub dataset_configs: HashMap, +} + +impl LiquidatorConfig { + pub fn load() -> eyre::Result { + // The path is relative to this file + // TODO : replace it with toml + let config_str = include_str!("./config.yaml"); + let config = serde_yaml::from_str(config_str)?; + Ok(config) + } + + pub fn is_event_enabled(&self, event_name: &str) -> bool { + self.enabled_events.get(event_name).cloned().unwrap_or(true) + } + + pub fn get(&self, key: &str) -> eyre::Result { + self.dataset_configs.get(key) + .ok_or_else(|| eyre::eyre!("No config found for {}", key)) + .and_then(|v| serde_yaml::from_value(v.clone()) + .map_err(|e| eyre::eyre!("Failed to parse config for {}: {}", key, e))) + } +} \ No newline at end of file diff --git a/crates/manager/src/strategy/mod.rs b/crates/manager/src/strategy/mod.rs index e028dcf..015e80a 100644 --- a/crates/manager/src/strategy/mod.rs +++ b/crates/manager/src/strategy/mod.rs @@ -1 +1,2 @@ pub mod path_finder; +pub mod liquidator; \ No newline at end of file From 4e315cffc2994945ca15caa33d1f33ddcb08b0d8 Mon Sep 17 00:00:00 2001 From: 0xwiederholen <0xwiederholen@gmail.com> Date: Tue, 12 Aug 2025 09:19:10 +0900 Subject: [PATCH 07/12] feat: frame for liquidator strategy --- Cargo.lock | 2 + Cargo.toml | 6 + crates/extension/src/exex.rs | 4 +- .../datasets/aave_execute_borrow.rs | 60 +++++++++ .../strategy/src/liquidator/datasets/mod.rs | 1 + crates/strategy/src/liquidator/db_writer.rs | 65 ++++++++++ crates/strategy/src/liquidator/liquidator.rs | 115 ++++++++++++++++++ 7 files changed, 251 insertions(+), 2 deletions(-) create mode 100644 crates/strategy/src/liquidator/datasets/aave_execute_borrow.rs create mode 100644 crates/strategy/src/liquidator/datasets/mod.rs create mode 100644 crates/strategy/src/liquidator/db_writer.rs create mode 100644 crates/strategy/src/liquidator/liquidator.rs diff --git a/Cargo.lock b/Cargo.lock index 275e5d0..873eabd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9464,8 +9464,10 @@ dependencies = [ "eyre", "reth-revm", "reth-tracing", + "rocksdb", "serde", "serde_json", + "serde_yaml", "tokio", "toml", ] diff --git a/Cargo.toml b/Cargo.toml index a06ea2d..1880026 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -86,3 +86,9 @@ rayon = "1.10.0" reqwest = { version = "0.11", default-features = false, features = ["json", "rustls-tls"] } chrono = { version = "0.4", default-features = false, features = ["clock"] } once_cell = "1" + +#rocksdb +rocksdb = "0.23.0" + +# todo: replace it with toml +serde_yaml = "0.9.34" \ No newline at end of file diff --git a/crates/extension/src/exex.rs b/crates/extension/src/exex.rs index 64b7b4b..c9133bd 100644 --- a/crates/extension/src/exex.rs +++ b/crates/extension/src/exex.rs @@ -18,8 +18,8 @@ use searcher_reth_strategy::core::strategy::Strategy; use tokio::sync::broadcast; use crate::relayer_pool::{ RelayerMessage, RelayerPool, WalletPool }; use searcher_reth_indexer::{indexer::Indexer}; -use alloy_consensus::{BlockHeader, Header, Block}; -use reth::primitives::{ EthereumHardforks, NodePrimitives}; +use alloy_consensus::{Header, Block}; +use reth::primitives::{NodePrimitives}; use searcher_reth_indexer::utils::Config; use searcher_reth_indexer::db_writer::RocksDB; diff --git a/crates/strategy/src/liquidator/datasets/aave_execute_borrow.rs b/crates/strategy/src/liquidator/datasets/aave_execute_borrow.rs new file mode 100644 index 0000000..089ae37 --- /dev/null +++ b/crates/strategy/src/liquidator/datasets/aave_execute_borrow.rs @@ -0,0 +1,60 @@ +use crate::db_writer::RocksDB; +use alloy_sol_types::{sol, SolEvent}; +use alloy_primitives::{address, Address}; +use reth_primitives::{RecoveredBlock, Receipt, Block}; +use eyre::Result; +use reth_node_api::FullNodeComponents; +use tracing::debug; +use crate::indexer::ProcessingComponents; +// Pool contarct address +const ARBITRUM_AAVE_CONTRACT_ADDRESS: Address = address!("0x794a61358D6845594F94dc1DB02A252b5b4814aD"); + +sol! { + event Borrow( + address indexed reserve, + address user, + address indexed onBehalfOf, + uint256 amount, + uint8 interestRateMode, + uint256 borrowRate, + uint16 indexed referralCode + ); +} + +pub async fn process_aave_execute_borrow( + block_data: &(RecoveredBlock, Vec), + components: ProcessingComponents, + db: &RocksDB, +) -> Result<()> { + let block = &block_data.0; + let receipts = &block_data.1; + + // Iterate through transactions and their receipts + for (tx_idx, (tx, receipt)) in block.body().transactions.iter().zip(receipts.iter()).enumerate() { + // Process each log in the receipt + for (log_idx, log) in receipt.logs.iter().enumerate() { + // Filter on dolomite contract address + if log.address == ARBITRUM_AAVE_CONTRACT_ADDRESS { + // First check if this log matches our event signature + if log.topics().get(0) != Some(&Borrow::SIGNATURE_HASH) { + continue; + } + + match Borrow::decode_raw_log(log.topics(), &log.data.data) { + Ok(create) => { + db.save( + "aave_borrow", + &format!("{}", create.onBehalfOf), + &format!("{}_{}", create.reserve, create.amount) + ); + } + Err(e) => { + debug!("Failed to decode aave borrow event: {:?}", e); + } + } + } + } + } + Ok(()) +} + diff --git a/crates/strategy/src/liquidator/datasets/mod.rs b/crates/strategy/src/liquidator/datasets/mod.rs new file mode 100644 index 0000000..8b75b16 --- /dev/null +++ b/crates/strategy/src/liquidator/datasets/mod.rs @@ -0,0 +1 @@ +pub mod aave_execute_borrow; \ No newline at end of file diff --git a/crates/strategy/src/liquidator/db_writer.rs b/crates/strategy/src/liquidator/db_writer.rs new file mode 100644 index 0000000..4105fd5 --- /dev/null +++ b/crates/strategy/src/liquidator/db_writer.rs @@ -0,0 +1,65 @@ +use std::sync::Arc; +use alloy_primitives::{Address, FixedBytes, Uint, Signed}; +use std::collections::{HashSet, HashMap}; +use rocksdb::{DB, ColumnFamily, Options, ColumnFamilyDescriptor}; + +#[derive(Clone)] +pub struct RocksDB { + db: Arc +} + +impl RocksDB { + pub fn init(file_path: &str, required_cfs: &[&str]) -> Self { + let existing_cfs = DB::list_cf(&Options::default(), file_path) + .unwrap_or_else(|_| vec!["default".to_string()]); + let existing_cf_set: HashSet = existing_cfs.iter().cloned().collect(); + + let cf_descriptors: Vec<_> = existing_cfs + .iter() + .map(|cf_name| ColumnFamilyDescriptor::new(cf_name, Options::default())) + .collect(); + + let mut db = DB::open_cf_descriptors(&Options::default(), file_path, cf_descriptors) + .expect("Failed to open DB"); + + for cf in required_cfs { + if !existing_cf_set.contains(*cf) { + db.create_cf(*cf, &Options::default()) + .expect(&format!("Failed to create CF: {}", cf)); + } + } + + RocksDB { + db: Arc::new(db) + } + } + + pub fn save(&self, cf: &str, k: &str, v: &str) -> bool { + let cf = self.db.cf_handle(cf).expect("missing CF"); + self.db.put_cf(cf, k.as_bytes(), v.as_bytes()).is_ok() + } + + pub fn find(&self, cf: &str, k: &str) -> Option { + let cf = self.db.cf_handle(cf).expect("missing CF"); + match self.db.get_cf(cf, k.as_bytes()) { + Ok(Some(v)) => { + let result = String::from_utf8(v).unwrap(); + println!("Finding '{}' returns '{}'", k, result); + Some(result) + }, + Ok(None) => { + println!("Finding '{}' returns None", k); + None + }, + Err(e) => { + println!("Error retrieving value for {}: {}", k, e); + None + } + } + } + + pub fn delete(&self, cf: &str, k: &str) -> bool { + let cf = self.db.cf_handle(cf).expect("missing CF"); + self.db.delete_cf(cf, k.as_bytes()).is_ok() + } +} \ No newline at end of file diff --git a/crates/strategy/src/liquidator/liquidator.rs b/crates/strategy/src/liquidator/liquidator.rs new file mode 100644 index 0000000..74af662 --- /dev/null +++ b/crates/strategy/src/liquidator/liquidator.rs @@ -0,0 +1,115 @@ +use crate::liquidator::db_writer::RocksDB; +use crate::liquidator::datasets::{ + aave_execute_borrow::process_aave_execute_borrow, +}; +use alloy_consensus::{Header}; +use eyre::Result; +use reth::builder::NodeTypes; +use reth::primitives::NodePrimitives; +use reth_primitives::{TransactionSigned, Receipt, RecoveredBlock, Block}; +use reth_node_api::{ConfigureEvm, FullNodeComponents, FullNodeTypes}; +use reth_tracing::tracing::{info, warn}; +use std::time::Instant; +use searcher_reth_manager::{ + common::{ CommonStrategyConfig, ONE_ETHER, StrategyConfig }, + gas::GasConfig +}; + +// Structure to hold all the components needed for processing +#[derive(Clone)] +pub struct ProcessingComponents { + pub provider: Node::Provider, + config: StrategyConfig, +} + +struct ProcessorInfo { + table_name: &'static str, + processor_name: &'static str, + processor: for<'a> fn( + &'a (RecoveredBlock, Vec), + ProcessingComponents, + &'a RocksDB + ) -> futures::future::BoxFuture<'a, Result<()>>, +} + +impl ProcessorInfo { + fn new( + table_name: &'static str, + processor_name: &'static str, + processor: for<'a> fn( + &'a (RecoveredBlock, Vec), + ProcessingComponents, + &'a RocksDB + ) -> futures::future::BoxFuture<'a, Result<()>>, + ) -> Self { + Self { + table_name, + processor_name, + processor, + } + } +} + +pub struct Liquidator { + processors: Vec>, + config: StrategyConfig, +} + +impl Strategy for Liquidator { + fn new(config: Config) -> Self { + let mut liquidator = Self { + processors: Vec::new(), + config, + }; + + liquidator.add_processor("aave_execute_borrow", "AaveExecuteBorrow"); + info!("Initialized liquidator with processors: {:?}", liquidator.list_processors()); + liquidator + } + + fn gas_config(&self) -> GasConfig { + self.config.get_gas_config() + } + + fn get_code(&self) -> Bytecode { + Bytecode::default() + } + + fn get_vault(&self) -> Address { + self.config.get_vault() + } + + fn prepare(&mut self, chain_id: u64) { + tracing::info!( + target: "liquidator", + event = "prepare_enter", + chain_id = chain_id, + "Entered prepare()" + ); + // TODO : define actions for prepare stage + tracing::info!( + target: "liquidator", + event = "prepare_exit", + chain_id = chain_id, + "Exiting prepare()" + ); + } + + fn find_profitable_candidates< + T: PoolTransaction, + DB: DBProvider + BlockHashReader + StateCommitmentProvider + >( + &mut self, + block: NumHash, + latest_state_provider: LatestStateProviderRef<'_, DB>, + pending_txs: Vec + ) -> Result, AccessList)>, Error> { + // TODO : define actions for find_profitable_candidates stage + // 1. get liquidations candidates from db + // 2. filter candidates which can be liquidated + // 3. return the candidates + // 4. store new candidates that indexer processed + } +} + + From e29b3737d41e2d0d0c2191ba82086ffcf5731749 Mon Sep 17 00:00:00 2001 From: 0xwiederholen <0xwiederholen@gmail.com> Date: Fri, 15 Aug 2025 14:38:33 +0900 Subject: [PATCH 08/12] wip : replacing indexerExEx with liquidatorExEx --- .DS_Store | Bin 8196 -> 6148 bytes Cargo.lock | 76 +---- Cargo.toml | 3 +- bin/searcher-reth/src/main.rs | 9 +- crates/extension/Cargo.toml | 1 - crates/extension/src/exex.rs | 169 ++++++----- crates/extension/src/lib.rs | 6 +- crates/indexer/Cargo.toml | 49 --- crates/indexer/config.yaml | 0 .../src/datasets/aave_execute_borrow.rs | 60 ---- crates/indexer/src/datasets/mod.rs | 2 - crates/indexer/src/db_writer.rs | 67 ---- crates/indexer/src/indexer.rs | 285 ------------------ crates/indexer/src/lib.rs | 4 - crates/indexer/src/utils.rs | 130 -------- crates/manager/src/common.rs | 31 +- crates/manager/src/strategy/liquidator/mod.rs | 75 +++-- crates/manager/src/types.rs | 12 + crates/strategy/Cargo.toml | 4 + crates/strategy/src/lib.rs | 1 + .../datasets/aave_execute_borrow.rs | 6 +- .../datasets/dolomite_borrow_position.rs | 6 +- .../strategy/src/liquidator/datasets/mod.rs | 3 +- crates/strategy/src/liquidator/db_writer.rs | 23 ++ crates/strategy/src/liquidator/liquidator.rs | 249 +++++++++++++-- crates/strategy/src/liquidator/mod.rs | 9 + crates/strategy/src/liquidator/types.rs | 10 + .../strategy/src/path_finding/path_finder.rs | 8 +- 28 files changed, 475 insertions(+), 823 deletions(-) delete mode 100644 crates/indexer/Cargo.toml delete mode 100644 crates/indexer/config.yaml delete mode 100644 crates/indexer/src/datasets/aave_execute_borrow.rs delete mode 100644 crates/indexer/src/datasets/mod.rs delete mode 100644 crates/indexer/src/db_writer.rs delete mode 100644 crates/indexer/src/indexer.rs delete mode 100644 crates/indexer/src/lib.rs delete mode 100644 crates/indexer/src/utils.rs rename crates/{indexer/src => strategy/src/liquidator}/datasets/dolomite_borrow_position.rs (94%) create mode 100644 crates/strategy/src/liquidator/mod.rs create mode 100644 crates/strategy/src/liquidator/types.rs diff --git a/.DS_Store b/.DS_Store index 51010bfb22acf7d72d39e11dd75c7eb5120a6f82..3916e7ab332e2850aa08cb94d7c1eb2503ad089e 100644 GIT binary patch delta 116 zcmZp1XfcprU|?W$DortDU=RQ@Ie-{MGjUE#6q~50$SANeU^gS9z-AtSV#dklBCQ*f ym{}IHb8rYU162Wm05_0u1!>rr_?>w&zl4l@8cc@WqD literal 8196 zcmeHMziSjh6n=BJT#lSV%9JYz{s^c6n^dmh6vQGzIt$5NF02Q)x2K7PO<0g%EG>mt z+6lo*1dG_jDi#)k_#ap)f}r4!@6C+2Gq<+}Qi$Rkn0d>*_vZWd+r62*4H2pNTb)^= zX(Fnlvz=;S7}MC#z0fLl<|wQ}KK0_+#pN^^6e49p6c7bO0Z~8{5C#4g1#o6_tCyVn z#w(*JAPW4K3h@3AqO*;x&5YGs2L@dNzy_Gs4cAkA0UKHyS(_OP51KNmuqIX65<{7E z%!f8EvNki;q?5A6hqB1ZRwzoNWBpKvlZuR$Q4|mbiVAS;9?&U@sY`2a{jM&byy*Ip zwApO;(l+MmukWAU`|@M=yzlS))<3wOe;ZKiJ4mG}wSn4qQHc)k<{lUG{f&)dS3YY@ zo0p_HV$tj3*}bSn%XE`kv_>nm>Z3ko9X!8_y>;)+v#;iUM`CgYWicKP^?~m@ppItI z@;KCXJfDkyPOBdn)n#(BtwsH#70>lF`WKx%fq|hi6s|4Bm($b63~l z9{OC&trOdO+rOR*s~Xd)i#pEl;m$2F@+6)`7rEx$>|&k%2nVWel*PGx?mT(C`TUc9 zCS%uhx$PJ0@id2V4R1>eEMvSm?p{%uS@`w>{5-$QYvG|v?$S%%FGgbW-%nYL!=rNM zc!)m?bd&Sn=pdsga8L!RCUKhU{|n{c{|~ynG@KMZjlB9wC?YcpdpgC-vWv eyre::Result<()> { // Install Exex for various strategies let searcher_exex = SearcherExEx::new(wallet, signal_manager.subscribe()); + let liquidator_exex = LiquidatorExEx::new(wallet, signal_manager.subscribe()); let mut node_builder = builder.node(EthereumNode::default()); @@ -48,7 +49,9 @@ fn main() -> eyre::Result<()> { // Install Indexer node_builder = node_builder.install_exex(LIQUIDATOR_EXEX_ID, { - IndexerExEx::init + let liquidator_cfg = config.clone().read().unwrap().get_strategy(LIQUIDATOR_EXEX_ID).unwrap(); + let liquidator = Liquidator::new(liquidator_cfg); + move |ctx| liquidator_exex.run(ctx, liquidator) }); // TODO: Add other strategies here as needed diff --git a/crates/extension/Cargo.toml b/crates/extension/Cargo.toml index c956453..138e9fc 100644 --- a/crates/extension/Cargo.toml +++ b/crates/extension/Cargo.toml @@ -42,7 +42,6 @@ clap.workspace = true searcher-reth-strategy.workspace = true searcher-reth-manager.workspace = true -searcher-reth-indexer.workspace = true rayon.workspace = true reth-transaction-pool.workspace = true diff --git a/crates/extension/src/exex.rs b/crates/extension/src/exex.rs index c9133bd..59b9c72 100644 --- a/crates/extension/src/exex.rs +++ b/crates/extension/src/exex.rs @@ -17,88 +17,115 @@ use searcher_reth_manager::SignalType; use searcher_reth_strategy::core::strategy::Strategy; use tokio::sync::broadcast; use crate::relayer_pool::{ RelayerMessage, RelayerPool, WalletPool }; -use searcher_reth_indexer::{indexer::Indexer}; use alloy_consensus::{Header, Block}; use reth::primitives::{NodePrimitives}; -use searcher_reth_indexer::utils::Config; -use searcher_reth_indexer::db_writer::RocksDB; -pub struct IndexerExEx; +pub struct LiquidatorExEx { + pub wallet: Arc, + pub signal_rx: broadcast::Receiver, +} -impl IndexerExEx { - pub async fn init( - ctx: ExExContext, - ) -> Result>> - where - Node: FullNodeComponents, - //::ChainSpec: EthereumHardforks, - ::Primitives: NodePrimitives< - BlockHeader = Header, - Block = Block, - Receipt = Receipt, - SignedTx = TransactionSigned - >, - { - let config = Config::load().wrap_err("Failed to load configuration")?; - - //let client = Arc::new(connect_to_postgres().await?); - //create_tables(&client).await?; - - let db = RocksDB::init("rocksdb", &["default"]); - - // Create indexer with all processors initialized internally - let indexer = Indexer::new(config); - - Ok(Self::indexer_exex(ctx, db, indexer)) +impl LiquidatorExEx { + pub fn new(wallet: Arc, signal_rx: broadcast::Receiver) -> Self { + Self { wallet, signal_rx } } - async fn indexer_exex( + pub async fn run( + self, mut ctx: ExExContext, - //client: Arc, - db: RocksDB, - indexer: Indexer, - ) -> Result<()> + mut strategy: S + ) -> Result>> where - Node::Types: NodeTypes, - //::ChainSpec: EthereumHardforks, - ::Primitives: NodePrimitives< - BlockHeader = Header, - Block = Block, - Receipt = Receipt, - SignedTx = TransactionSigned - >, + S: Strategy, + Node: FullNodeComponents, + Node::Pool: TransactionPool, + <::Provider as DatabaseProviderFactory>::Provider: BlockHashReader + + StateCommitmentProvider, + Node::Provider: BlockReaderIdExt + ReceiptProvider + AccountReader + ChainSpecProvider { - // Process all new chain state notifications - while let Some(notification) = ctx.notifications.try_next().await? { - match ¬ification { - ExExNotification::ChainCommitted { new } => { - // Process the committed blocks - if let Err(e) = indexer.process_blocks( - new.blocks_and_receipts(), - //&client, - &db, - ctx.provider().clone(), - //Arc::new(ctx.evm_config().clone()), - //Arc::new(ctx.pool().clone()), - //Arc::new(ctx.network().clone()), - ).await { - tracing::warn!("Failed to process committed blocks: {}", e); + let wallet = self.wallet.clone(); + let signal_rx = self.signal_rx.resubscribe(); + let vault = strategy.get_vault(); + Ok(async move { + let relayer_pool = Arc::new( + RelayerPool::new( + ctx.components.clone(), + wallet, + signal_rx, + strategy.gas_config() + ).await? + ); + let relayer_tx = relayer_pool.start().await?; + tracing::info!( + target: "reth-exex", + event = "relayer_pool_started", + "Starting Relayer Pool" + ); + strategy.prepare(ctx.components.network().chain_id()); + while let Some(notification) = ctx.notifications.next().await { + if let Ok(ExExNotification::ChainCommitted { new: chain }) = notification { + let block = chain.tip(); + let num_hash = block.num_hash(); + let bytecode = strategy.get_code(); + + let database_provider: <::Provider as DatabaseProviderFactory>::Provider = ctx + .provider() + .database_provider_ro()?; + let latest_state_provider = LatestStateProviderRef::new(&database_provider); + + // 1. find candidates to liquidate + // pending txs is not used for liquidatior strategy. pass empty vectors. + let profitable_candidates = strategy.find_profitable_candidates( + num_hash, + latest_state_provider, + Vec::::new() + )?; + + tracing::info!( + target: "reth-exex", + event = "filter_candidates", + success = profitable_candidates.is_none(), + num_hash = ?num_hash, + "Profitable candidates found" + ); + + if profitable_candidates.is_none() { + ctx.events.send(ExExEvent::FinishedHeight(num_hash))?; + continue; } - - // Advance ExEx - ctx.events.send(ExExEvent::FinishedHeight(new.tip().num_hash()))?; - }, - ExExNotification::ChainReorged { .. } => { - // do nothing for reorg - continue; - }, - ExExNotification::ChainReverted { .. } => { - // do nothing for revert - continue; - }, + + // 2. send candidates to relayer pool for braodcasting trnasactions + let (calldata, access_list) = profitable_candidates.unwrap(); + let relayer_channel = relayer_tx.clone(); + tokio::spawn(async move { + let message = RelayerMessage { to: vault, calldata, access_list }; + let r = relayer_channel.send(message).await; + + match r { + Ok(_) => + tracing::info!( + target: "reth-exex", + event = "send_candidates_to_relayer_pool", + success = true, + num_hash = ?num_hash, + "Successfully sent candidates to relayer pool" + ), + Err(e) => + tracing::error!( + target: "reth-exex", + event = "send_candidates_to_relayer_pool", + success = false, + error = ?e, + num_hash = ?num_hash, + "Failed to send candidates to relayer pool" + ), + } + }); + ctx.events.send(ExExEvent::FinishedHeight(num_hash))?; + } } - } - Ok(()) + Ok(()) + }) } } diff --git a/crates/extension/src/lib.rs b/crates/extension/src/lib.rs index 2b625f2..90e0f10 100644 --- a/crates/extension/src/lib.rs +++ b/crates/extension/src/lib.rs @@ -7,8 +7,4 @@ pub mod util { pub mod strategy { pub use searcher_reth_strategy::*; -} - -pub mod indexer { - pub use searcher_reth_indexer::*; -} +} \ No newline at end of file diff --git a/crates/indexer/Cargo.toml b/crates/indexer/Cargo.toml deleted file mode 100644 index a76762d..0000000 --- a/crates/indexer/Cargo.toml +++ /dev/null @@ -1,49 +0,0 @@ -[package] -name = "searcher-reth-indexer" -version = "0.0.0" -publish = false -edition.workspace = true -license.workspace = true - -[dependencies] -# reth -reth-chainspec.workspace = true -reth-evm.workspace = true -reth-execution-errors.workspace = true -reth-execution-types.workspace = true -reth-exex.workspace = true -reth-node-api.workspace = true -reth-node-ethereum.workspace = true -reth-primitives.workspace = true -reth-provider.workspace = true -reth-revm.workspace = true -reth-tracing.workspace = true -reth.workspace = true - -# alloy -alloy-consensus = { workspace = true, features = ["k256"] } -alloy-eips.workspace = true -alloy-genesis.workspace = true -alloy-primitives.workspace = true -alloy-rlp.workspace = true -alloy-sol-types.workspace = true -alloy-network.workspace = true -alloy-signer-local.workspace = true -alloy-rpc-types.workspace = true - -# async -futures-util.workspace = true -tokio.workspace = true - -reth-transaction-pool.workspace = true - -#tokio-postgres = { version = "0.7.13", features = ["with-chrono-0_4"] } -serde = { workspace = true, features = ["derive"] } -lazy_static = "1.5.0" -eyre.workspace = true -futures = "0.3.31" -chrono = "0.4.40" -serde_yaml = "0.9.34" -primitive-types = { version = "0.13.1", features = ["serde"] } -tracing = "0.1.41" -rocksdb = "0.23.0" diff --git a/crates/indexer/config.yaml b/crates/indexer/config.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/crates/indexer/src/datasets/aave_execute_borrow.rs b/crates/indexer/src/datasets/aave_execute_borrow.rs deleted file mode 100644 index 089ae37..0000000 --- a/crates/indexer/src/datasets/aave_execute_borrow.rs +++ /dev/null @@ -1,60 +0,0 @@ -use crate::db_writer::RocksDB; -use alloy_sol_types::{sol, SolEvent}; -use alloy_primitives::{address, Address}; -use reth_primitives::{RecoveredBlock, Receipt, Block}; -use eyre::Result; -use reth_node_api::FullNodeComponents; -use tracing::debug; -use crate::indexer::ProcessingComponents; -// Pool contarct address -const ARBITRUM_AAVE_CONTRACT_ADDRESS: Address = address!("0x794a61358D6845594F94dc1DB02A252b5b4814aD"); - -sol! { - event Borrow( - address indexed reserve, - address user, - address indexed onBehalfOf, - uint256 amount, - uint8 interestRateMode, - uint256 borrowRate, - uint16 indexed referralCode - ); -} - -pub async fn process_aave_execute_borrow( - block_data: &(RecoveredBlock, Vec), - components: ProcessingComponents, - db: &RocksDB, -) -> Result<()> { - let block = &block_data.0; - let receipts = &block_data.1; - - // Iterate through transactions and their receipts - for (tx_idx, (tx, receipt)) in block.body().transactions.iter().zip(receipts.iter()).enumerate() { - // Process each log in the receipt - for (log_idx, log) in receipt.logs.iter().enumerate() { - // Filter on dolomite contract address - if log.address == ARBITRUM_AAVE_CONTRACT_ADDRESS { - // First check if this log matches our event signature - if log.topics().get(0) != Some(&Borrow::SIGNATURE_HASH) { - continue; - } - - match Borrow::decode_raw_log(log.topics(), &log.data.data) { - Ok(create) => { - db.save( - "aave_borrow", - &format!("{}", create.onBehalfOf), - &format!("{}_{}", create.reserve, create.amount) - ); - } - Err(e) => { - debug!("Failed to decode aave borrow event: {:?}", e); - } - } - } - } - } - Ok(()) -} - diff --git a/crates/indexer/src/datasets/mod.rs b/crates/indexer/src/datasets/mod.rs deleted file mode 100644 index 6b72da2..0000000 --- a/crates/indexer/src/datasets/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod dolomite_borrow_position; -pub mod aave_execute_borrow; \ No newline at end of file diff --git a/crates/indexer/src/db_writer.rs b/crates/indexer/src/db_writer.rs deleted file mode 100644 index 9ab033e..0000000 --- a/crates/indexer/src/db_writer.rs +++ /dev/null @@ -1,67 +0,0 @@ -use eyre::Result; -use std::sync::Arc; -use futures::pin_mut; -use alloy_primitives::{Address, FixedBytes, Uint, Signed}; -use std::collections::{HashSet, HashMap}; -use rocksdb::{DB, ColumnFamily, Options, ColumnFamilyDescriptor}; - -#[derive(Clone)] -pub struct RocksDB { - db: Arc -} - -impl RocksDB { - pub fn init(file_path: &str, required_cfs: &[&str]) -> Self { - let existing_cfs = DB::list_cf(&Options::default(), file_path) - .unwrap_or_else(|_| vec!["default".to_string()]); - let existing_cf_set: HashSet = existing_cfs.iter().cloned().collect(); - - let cf_descriptors: Vec<_> = existing_cfs - .iter() - .map(|cf_name| ColumnFamilyDescriptor::new(cf_name, Options::default())) - .collect(); - - let mut db = DB::open_cf_descriptors(&Options::default(), file_path, cf_descriptors) - .expect("Failed to open DB"); - - for cf in required_cfs { - if !existing_cf_set.contains(*cf) { - db.create_cf(*cf, &Options::default()) - .expect(&format!("Failed to create CF: {}", cf)); - } - } - - RocksDB { - db: Arc::new(db) - } - } - - pub fn save(&self, cf: &str, k: &str, v: &str) -> bool { - let cf = self.db.cf_handle(cf).expect("missing CF"); - self.db.put_cf(cf, k.as_bytes(), v.as_bytes()).is_ok() - } - - pub fn find(&self, cf: &str, k: &str) -> Option { - let cf = self.db.cf_handle(cf).expect("missing CF"); - match self.db.get_cf(cf, k.as_bytes()) { - Ok(Some(v)) => { - let result = String::from_utf8(v).unwrap(); - println!("Finding '{}' returns '{}'", k, result); - Some(result) - }, - Ok(None) => { - println!("Finding '{}' returns None", k); - None - }, - Err(e) => { - println!("Error retrieving value for {}: {}", k, e); - None - } - } - } - - pub fn delete(&self, cf: &str, k: &str) -> bool { - let cf = self.db.cf_handle(cf).expect("missing CF"); - self.db.delete_cf(cf, k.as_bytes()).is_ok() - } -} \ No newline at end of file diff --git a/crates/indexer/src/indexer.rs b/crates/indexer/src/indexer.rs deleted file mode 100644 index 3ae1346..0000000 --- a/crates/indexer/src/indexer.rs +++ /dev/null @@ -1,285 +0,0 @@ -use crate::utils::Config; -use crate::db_writer::RocksDB; -use crate::datasets::{ - dolomite_borrow_position::process_dolomite_borrow_positions, - aave_execute_borrow::process_aave_execute_borrow, -}; -use alloy_consensus::{Header}; -use alloy_rpc_types::{BlockId, BlockNumberOrTag}; -use eyre::Result; -use reth::builder::NodeTypes; -use reth::primitives::{EthereumHardforks, NodePrimitives}; -use reth_primitives::{TransactionSigned, Receipt, RecoveredBlock, Block}; -use reth_node_api::{ConfigureEvm, FullNodeComponents, FullNodeTypes}; -use reth_tracing::tracing::{info, warn}; -use std::{sync::Arc, time::Instant, collections::HashSet}; - -// Structure to hold all the components needed for processing -#[derive(Clone)] -pub struct ProcessingComponents { - // pub eth_api: Arc>, - // pub block_traces: Option>, - pub provider: Node::Provider, - //pub client: Arc, - pub config: Config, -} - -struct ProcessorInfo { - table_name: &'static str, - processor_name: &'static str, - processor: for<'a> fn( - &'a (RecoveredBlock, Vec), - ProcessingComponents, - &'a RocksDB - ) -> futures::future::BoxFuture<'a, Result<()>>, -} - -impl ProcessorInfo { - fn new( - table_name: &'static str, - processor_name: &'static str, - processor: for<'a> fn( - &'a (RecoveredBlock, Vec), - ProcessingComponents, - &'a RocksDB - ) -> futures::future::BoxFuture<'a, Result<()>>, - ) -> Self { - Self { - table_name, - processor_name, - processor, - } - } -} - -pub struct Indexer { - processors: Vec>, - config: Config, -} - -impl Indexer -where - Node: FullNodeComponents + FullNodeTypes, - Node::Types: NodeTypes, - //::ChainSpec: EthereumHardforks, - ::Primitives: NodePrimitives< - BlockHeader = Header, - Block = Block, - Receipt = Receipt, - SignedTx = TransactionSigned, - >, - Node::Provider: reth::providers::BlockReader> - + reth::providers::HeaderProvider
- + reth::providers::ReceiptProvider - + reth::providers::TransactionsProvider, - Node::Evm: ConfigureEvm - //EthApi: Call + LoadPendingBlock + FullEthApiTypes, -{ - pub fn new(config: Config) -> Self { - let mut indexer = Self { - processors: Vec::new(), - config, - }; - - // Register all available processors - // e.g) indexer.add_processor("headers", "Headers"); - indexer.add_processor("dolomite_borrow_positions", "DolomiteBorrowPositions"); - //indexer.add_processor("aave_borrow", "AaveBorrow"); - - info!("Initialized indexer with processors: {:?}", indexer.list_processors()); - indexer - } - - pub fn add_processor(&mut self, table_name: &'static str, processor_name: &'static str) { - let processor = match table_name { - "dolomite_borrow_positions" => ProcessorInfo::new( - table_name, - processor_name, - |block_data, components, db| Box::pin(process_dolomite_borrow_positions::(block_data, components, db)) - ), - // "aave_borrow" => ProcessorInfo::new( - // table_name, - // processor_name, - // |block_data, components, db| Box::pin(process_aave_execute_borrow::(block_data, components, db)) - // ), - _ => return, // Skip unknown processors - }; - self.processors.push(processor); - } - - pub fn list_processors(&self) -> Vec<&str> { - self.processors.iter().map(|p| p.processor_name).collect() - } - - pub async fn process_blocks( - &self, - blocks_and_receipts: impl Iterator, &Vec)>, - //client: &Arc, - db: &RocksDB, - provider: Node::Provider, - //evm_config: Arc, - //pool: Arc, - //network: Arc, - ) -> Result<()> - where - Node::Evm: ConfigureEvm - //EthApi: Call + LoadPendingBlock + FullEthApiTypes, - { - // Convert the iterator items into owned values directly - let blocks_and_receipts: Vec<_> = blocks_and_receipts - .map(|(block, receipts)| (block.clone(), receipts.clone())) - .collect(); - - for (block, receipts) in blocks_and_receipts { - let block_number = block.number; - //let block_id = BlockId::Number(BlockNumberOrTag::from(block_number)); - - // Create EthAPI - // let eth_api = crate::utils::create_eth_api::( - // provider.clone(), - // (*evm_config).clone(), - // (*pool).clone(), - // (*network).clone() - // ); - - // // Create TraceAPI - // let trace_api = crate::utils::create_trace_api::( - // provider.clone(), - // (*evm_config).clone(), - // (*pool).clone(), - // (*network).clone() - // ); - - // // Get traces once for the block - // let block_traces = match trace_api.replay_block_transactions( - // block_id, - // HashSet::from_iter(vec![TraceType::Trace]) - // ).await { - // Ok(traces) => traces, - // Err(e) => { - // warn!("Failed to get traces for block {}: {}", block_number, e); - // None - // } - // }; - - // Create components for processing - let components = ProcessingComponents { - // eth_api: eth_api.clone(), - // block_traces, - provider: provider.clone(), - //client: Arc::clone(client), - config: self.config.clone(), - }; - - let block_data = (block, receipts); - if let Err(e) = self.process_block_data(&block_data, components, db).await { - warn!("Failed to process block {}: {}", block_number, e); - } - } - - Ok(()) - } - - pub async fn process_block_data( - &self, - block_data: &(RecoveredBlock, Vec), - components: ProcessingComponents, - db: &RocksDB - ) -> Result<()> - where - Node::Types: NodeTypes, - //::ChainSpec: EthereumHardforks, - ::Primitives: NodePrimitives< - BlockHeader = Header, - Block = Block, - Receipt = Receipt, - SignedTx = TransactionSigned - >, - { - let block_number = block_data.0.number; - - // Create a vector to store all processing tasks - let mut tasks = Vec::new(); - - // Spawn a task for each enabled processor - for processor in &self.processors { - if self.config.is_event_enabled(processor.processor_name) { - // Clone necessary data for the task - let processor_name = processor.processor_name; - let processor_fn = processor.processor; - let block_data = block_data.clone(); - let components = components.clone(); - let db = db.clone(); // Clone db before spawn - // let table = get_table(processor.table_name) - // .expect(&format!("Table definition not found for {}", processor.table_name)); - - // Spawn the task - let task = tokio::spawn(async move { - let event_start_time = Instant::now(); - match processor_fn(&block_data, components, &db).await { - Ok(()) => Ok((processor_name, event_start_time.elapsed())), - Err(e) => Err((processor_name, e.to_string())) - } - }); - - tasks.push(task); - } - } - - // Wait for all tasks to complete and collect results - let mut total_records = 0; - let mut event_results = Vec::new(); - let mut failed_events = Vec::new(); - - for task in tasks { - match task.await { - Ok(Ok((name, duration))) => { - total_records += 1; // Assuming each event writes one record for now - event_results.push((name, duration)); - } - Ok(Err((name, error))) => { - failed_events.push((name, error)); - } - Err(e) => { - warn!("Task join error: {}", e); - } - } - } - - // Sort events by name for consistent logging - event_results.sort_by(|a, b| a.0.cmp(&b.0)); - - // Create a consolidated success log - if !event_results.is_empty() { - let events_summary: Vec = event_results - .iter() - .map(|(name, time)| { - format!("{}({}, {:.2}s)", name, 1, time.as_secs_f64()) - }) - .collect(); - - info!( - "exex{{id=\"exex-indexer\"}}: Block {} processed - Events: [{}], Total records: {}", - block_number, - events_summary.join(", "), - total_records, - ); - } - - // Create a consolidated error log - if !failed_events.is_empty() { - let failure_summary: Vec = failed_events - .iter() - .map(|(name, error)| format!("{}: {}", name, error)) - .collect(); - - warn!( - "exex{{id=\"exex-indexer\"}}: Block {} failures - {}", - block_number, - failure_summary.join(", ") - ); - } - - Ok(()) - } -} \ No newline at end of file diff --git a/crates/indexer/src/lib.rs b/crates/indexer/src/lib.rs deleted file mode 100644 index 9f1952e..0000000 --- a/crates/indexer/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub mod indexer; -pub mod db_writer; -pub mod utils; -pub mod datasets; \ No newline at end of file diff --git a/crates/indexer/src/utils.rs b/crates/indexer/src/utils.rs deleted file mode 100644 index 68e0e43..0000000 --- a/crates/indexer/src/utils.rs +++ /dev/null @@ -1,130 +0,0 @@ -// use alloy_eips::eip1559::ETHEREUM_BLOCK_GAS_LIMIT; -// use reth::builder::NodeTypes; -// use reth_node_api::FullNodeComponents; -// use reth_rpc::{EthApi, TraceApi}; -// use reth_rpc_eth_api::helpers::{Call, LoadPendingBlock}; -// use reth_rpc_eth_types::{EthStateCache, GasPriceOracle, FeeHistoryCache, FeeHistoryCacheConfig}; -// use reth_rpc_server_types::constants::{ -// DEFAULT_ETH_PROOF_WINDOW, -// DEFAULT_MAX_SIMULATE_BLOCKS, -// DEFAULT_PROOF_PERMITS, -// }; -// use reth_tasks::pool::{BlockingTaskGuard, BlockingTaskPool}; -use reth_tracing::tracing::info; -use serde::{Deserialize, Serialize}; -use std::{sync::Arc, collections::HashMap, env}; -use rocksdb::{DB, Options, ColumnFamilyDescriptor, ColumnFamily, DBWithThreadMode, SingleThreaded}; -use std::collections::HashSet; - -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct Config { - pub enabled_events: HashMap, - #[serde(flatten)] - pub dataset_configs: HashMap, -} - -impl Config { - pub fn load() -> eyre::Result { - // The path is relative to this file - let config_str = include_str!("../config.yaml"); - let config = serde_yaml::from_str(config_str)?; - Ok(config) - } - - pub fn is_event_enabled(&self, event_name: &str) -> bool { - self.enabled_events.get(event_name).cloned().unwrap_or(true) - } - - pub fn get(&self, key: &str) -> eyre::Result { - self.dataset_configs.get(key) - .ok_or_else(|| eyre::eyre!("No config found for {}", key)) - .and_then(|v| serde_yaml::from_value(v.clone()) - .map_err(|e| eyre::eyre!("Failed to parse config for {}: {}", key, e))) - } -} - -// Create a trace API instance with all the required components and trait bounds -// pub fn create_trace_api( -// provider: Node::Provider, -// evm_config: Node::Evm, -// pool: Node::Pool, -// network: Node::Network, -// ) -> Arc>> -// where -// Node: FullNodeComponents, -// Node::Types: NodeTypes, -// EthApi: Call + LoadPendingBlock, -// { -// let cache = EthStateCache::spawn( -// provider.clone(), -// Default::default(), -// ); - -// let fee_history_cache = FeeHistoryCache::new(FeeHistoryCacheConfig::default()); - -// let gas_oracle = GasPriceOracle::new( -// provider.clone(), -// Default::default(), -// cache.clone() -// ); - -// let eth_api = EthApi::new( -// provider.clone(), -// pool, -// network, -// cache.clone(), -// gas_oracle, -// ETHEREUM_BLOCK_GAS_LIMIT, -// DEFAULT_MAX_SIMULATE_BLOCKS, -// DEFAULT_ETH_PROOF_WINDOW, -// BlockingTaskPool::build().expect("failed to build tracing pool"), -// fee_history_cache, -// evm_config.clone(), -// DEFAULT_PROOF_PERMITS, -// ); - -// Arc::new(TraceApi::new( -// eth_api, -// BlockingTaskGuard::new(10), -// )) -// } - -// pub(crate) fn create_eth_api( -// provider: Node::Provider, -// evm_config: Node::Evm, -// pool: Node::Pool, -// network: Node::Network, -// ) -> Arc> -// where -// Node: FullNodeComponents, -// Node::Types: NodeTypes, -// EthApi: Call + LoadPendingBlock, -// { -// let cache = EthStateCache::spawn( -// provider.clone(), -// Default::default(), -// ); - -// let fee_history_cache = FeeHistoryCache::new(FeeHistoryCacheConfig::default()); - -// let gas_oracle = GasPriceOracle::new( -// provider.clone(), -// Default::default(), -// cache.clone() -// ); - -// Arc::new(EthApi::new( -// provider.clone(), -// pool.clone(), -// network.clone(), -// cache.clone(), -// gas_oracle, -// ETHEREUM_BLOCK_GAS_LIMIT, -// DEFAULT_MAX_SIMULATE_BLOCKS, -// DEFAULT_ETH_PROOF_WINDOW, -// BlockingTaskPool::build().expect("failed to build tracing pool"), -// fee_history_cache, -// evm_config.clone(), -// DEFAULT_PROOF_PERMITS, -// )) -// } \ No newline at end of file diff --git a/crates/manager/src/common.rs b/crates/manager/src/common.rs index 7a13918..7c2eb83 100644 --- a/crates/manager/src/common.rs +++ b/crates/manager/src/common.rs @@ -3,7 +3,11 @@ use reth_revm::state::Bytecode; use reth_tracing::tracing; use serde::{Deserialize, Serialize}; -use crate::{gas::GasConfig, strategy::{path_finder::PathFinderConfig, liquidator::LiquidatorConfig}, types::CandidateEntry}; +use crate::{ + gas::GasConfig, + strategy::{path_finder::PathFinderConfig, liquidator::LiquidatorConfig}, + types::StrategyCandidates +}; // Strategy configuration pub const PATH_FINDER_EXEX_ID: &str = "path-finder"; @@ -26,7 +30,7 @@ pub trait CommonStrategyConfig { fn get_contract(&self) -> Bytecode; fn get_profit_range(&self) -> (U256, U256); fn get_gas_config(&self) -> GasConfig; - fn load_candidates(&self, chain_id: u64) -> Vec; + fn load_candidates(&self, chain_id: u64) -> StrategyCandidates; } pub const ONE_ETHER: u128 = 1_000_000_000_000_000_000; @@ -34,9 +38,8 @@ pub const ONE_ETHER: u128 = 1_000_000_000_000_000_000; impl CommonStrategyConfig for StrategyConfig { fn get_exex_id(&self) -> &'static str { match self { - StrategyConfig::PathFinder(_) => PATH_FINDER_EXEX_ID, - StrategyConfig::Liquidator(_) => LIQUIDATOR_EXEX_ID, - // TODO: Add other strategy configurations + StrategyConfig::PathFinder(_) => "path-finder", + StrategyConfig::Liquidator(_) => "liquidator", } } @@ -151,7 +154,7 @@ impl CommonStrategyConfig for StrategyConfig { } } - fn load_candidates(&self, chain_id: u64) -> Vec { + fn load_candidates(&self, chain_id: u64) -> StrategyCandidates { match self { StrategyConfig::PathFinder(config) => { match config.load_candidates(chain_id) { @@ -161,17 +164,25 @@ impl CommonStrategyConfig for StrategyConfig { candidates.len(), chain_id ); - candidates + StrategyCandidates::Candidates(candidates) } Err(e) => { tracing::error!("Failed to load candidates: {}", e); - vec![] // Return an empty vector on error + StrategyCandidates::Candidates(vec![]) // Return an empty vector on error } } }, StrategyConfig::Liquidator(config) => { - // TODO: what candidates to load for liquidator? - vec![] + match config.load_processors(chain_id) { + Ok(processors) => { + tracing::info!("Loaded {} processors for chain_id: {}", processors.len(), chain_id); + StrategyCandidates::Processors(processors) + } + Err(e) => { + tracing::error!("Failed to load processors: {}", e); + StrategyCandidates::Processors(vec![]) // Return an empty vector on error + } + } } } } diff --git a/crates/manager/src/strategy/liquidator/mod.rs b/crates/manager/src/strategy/liquidator/mod.rs index 05384f1..cf9475a 100644 --- a/crates/manager/src/strategy/liquidator/mod.rs +++ b/crates/manager/src/strategy/liquidator/mod.rs @@ -1,42 +1,61 @@ +use std::{collections::HashMap, fs::File, io::BufReader, path::PathBuf}; +use eyre::eyre; use serde::{Deserialize, Serialize}; -use std::{collections::HashMap}; -use rocksdb::{DB, Options, ColumnFamilyDescriptor, ColumnFamily, DBWithThreadMode, SingleThreaded}; +use crate::types::ProcessorEntry; use crate::gas::GasConfig; -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] pub struct LiquidatorConfig { pub vault: String, pub contract: String, - - // gas configuration pub gas_config: GasConfig, - + pub path: PathBuf, + + pub max_liquidity: String, + /* Maximum liquidity to use for path finding ex. 1000 * + * 1ether(1000 USDC) */ + pub min_liquidity: String, + /* Minimum liquidity to use for path finding ex. 100 * + * 1ether(100 USDC) */ pub max_profit: String, pub min_profit: String, - - - pub enabled_events: HashMap, - #[serde(flatten)] - pub dataset_configs: HashMap, } -impl LiquidatorConfig { - pub fn load() -> eyre::Result { - // The path is relative to this file - // TODO : replace it with toml - let config_str = include_str!("./config.yaml"); - let config = serde_yaml::from_str(config_str)?; - Ok(config) - } - - pub fn is_event_enabled(&self, event_name: &str) -> bool { - self.enabled_events.get(event_name).cloned().unwrap_or(true) - } +// { +// "1": [ // chain_id = 1 +// { +// "table": "aave_execute_borrow", +// "processor": "AaveExecuteBorrow" +// }, +// { +// "table": "another_table", +// "processor": "AnotherProcessor" +// } +// ], +// "42161": [ // chain_id = 42161(Arbitrum) +// { +// "table": "dolomite_borrow", +// "processor": "DolomiteBorrow" +// } +// ] +// } - pub fn get(&self, key: &str) -> eyre::Result { - self.dataset_configs.get(key) - .ok_or_else(|| eyre::eyre!("No config found for {}", key)) - .and_then(|v| serde_yaml::from_value(v.clone()) - .map_err(|e| eyre::eyre!("Failed to parse config for {}: {}", key, e))) +impl LiquidatorConfig { + pub fn load_processors(&self, chain_id: u64) -> eyre::Result> { + if !self.path.exists() { + return Err(eyre!("Routes JSON file not found at: {}", self.path.display())); + } + + let file = File::open(&self.path)?; + let processors: HashMap> = + serde_json::from_reader(BufReader::new(file)) + .map_err(|e| eyre!("Failed to parse routes JSON: {}", e))?; + + let chain_processors = processors + .get(&chain_id.to_string()) + .ok_or_else(|| eyre!("No processors found for chain_id: {}", chain_id))?; + + Ok(chain_processors.clone()) } } \ No newline at end of file diff --git a/crates/manager/src/types.rs b/crates/manager/src/types.rs index 3a93254..0a2cf1b 100644 --- a/crates/manager/src/types.rs +++ b/crates/manager/src/types.rs @@ -36,6 +36,18 @@ pub struct CandidateEntry { pub initial_token: Address, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ProcessorEntry { + pub table: String, + pub processor: String, +} + +#[derive(Debug, Clone)] +pub enum StrategyCandidates { + Candidates(Vec), + Processors(Vec), +} + pub type CandidateMap = HashMap>; #[cfg(test)] diff --git a/crates/strategy/Cargo.toml b/crates/strategy/Cargo.toml index f8c27e8..b0fd138 100644 --- a/crates/strategy/Cargo.toml +++ b/crates/strategy/Cargo.toml @@ -34,6 +34,7 @@ alloy-rpc-types.workspace = true # async futures-util.workspace = true +futures.workspace = true tokio.workspace = true # misc @@ -44,6 +45,9 @@ reth-transaction-pool.workspace = true #searcher-reth searcher-reth-manager.workspace = true +#rocksdb +rocksdb.workspace = true + serde_json.workspace = true reqwest.workspace = true chrono.workspace = true diff --git a/crates/strategy/src/lib.rs b/crates/strategy/src/lib.rs index 87f78a2..8fbf17e 100644 --- a/crates/strategy/src/lib.rs +++ b/crates/strategy/src/lib.rs @@ -1,3 +1,4 @@ pub mod core; pub mod path_finding; +pub mod liquidator; pub mod profit_reporter; diff --git a/crates/strategy/src/liquidator/datasets/aave_execute_borrow.rs b/crates/strategy/src/liquidator/datasets/aave_execute_borrow.rs index 089ae37..60a5890 100644 --- a/crates/strategy/src/liquidator/datasets/aave_execute_borrow.rs +++ b/crates/strategy/src/liquidator/datasets/aave_execute_borrow.rs @@ -1,11 +1,11 @@ -use crate::db_writer::RocksDB; +use crate::liquidator::db_writer::RocksDB; use alloy_sol_types::{sol, SolEvent}; use alloy_primitives::{address, Address}; use reth_primitives::{RecoveredBlock, Receipt, Block}; use eyre::Result; use reth_node_api::FullNodeComponents; -use tracing::debug; -use crate::indexer::ProcessingComponents; +use reth_tracing::tracing::debug; +use crate::liquidator::ProcessingComponents; // Pool contarct address const ARBITRUM_AAVE_CONTRACT_ADDRESS: Address = address!("0x794a61358D6845594F94dc1DB02A252b5b4814aD"); diff --git a/crates/indexer/src/datasets/dolomite_borrow_position.rs b/crates/strategy/src/liquidator/datasets/dolomite_borrow_position.rs similarity index 94% rename from crates/indexer/src/datasets/dolomite_borrow_position.rs rename to crates/strategy/src/liquidator/datasets/dolomite_borrow_position.rs index e3bda17..bcb5e17 100644 --- a/crates/indexer/src/datasets/dolomite_borrow_position.rs +++ b/crates/strategy/src/liquidator/datasets/dolomite_borrow_position.rs @@ -1,11 +1,11 @@ -use crate::db_writer::RocksDB; +use crate::liquidator::db_writer::RocksDB; use alloy_sol_types::{sol, SolEvent}; use alloy_primitives::{address, Address}; use reth_primitives::{RecoveredBlock, Receipt, Block}; use eyre::Result; use reth_node_api::FullNodeComponents; -use tracing::debug; -use crate::indexer::ProcessingComponents; +use reth_tracing::tracing::debug; +use crate::liquidator::ProcessingComponents; // BorrowPositionProxyV2 address const BERA_DOLOMITE_CONTRACT_ADDRESS: Address = address!("0xC06271eb97d960F4034DDF953e16271CcB2B10BD"); diff --git a/crates/strategy/src/liquidator/datasets/mod.rs b/crates/strategy/src/liquidator/datasets/mod.rs index 8b75b16..7d0b52f 100644 --- a/crates/strategy/src/liquidator/datasets/mod.rs +++ b/crates/strategy/src/liquidator/datasets/mod.rs @@ -1 +1,2 @@ -pub mod aave_execute_borrow; \ No newline at end of file +pub mod aave_execute_borrow; +pub mod dolomite_borrow_position; \ No newline at end of file diff --git a/crates/strategy/src/liquidator/db_writer.rs b/crates/strategy/src/liquidator/db_writer.rs index 4105fd5..45d6916 100644 --- a/crates/strategy/src/liquidator/db_writer.rs +++ b/crates/strategy/src/liquidator/db_writer.rs @@ -3,6 +3,29 @@ use alloy_primitives::{Address, FixedBytes, Uint, Signed}; use std::collections::{HashSet, HashMap}; use rocksdb::{DB, ColumnFamily, Options, ColumnFamilyDescriptor}; +// database table 이름 타입으로 정의 +pub enum TableName { + DolomiteBorrowPositions, + AaveExecuteBorrow, +} + +impl TableName { + pub fn from_str(s: &str) -> Option { + match s { + "dolomite_borrow_positions" => Some(TableName::DolomiteBorrowPositions), + "aave_execute_borrow" => Some(TableName::AaveExecuteBorrow), + _ => None + } + } + + pub fn as_str(&self) -> &'static str { + match self { + TableName::DolomiteBorrowPositions => "dolomite_borrow_positions", + TableName::AaveExecuteBorrow => "aave_execute_borrow", + } + } +} + #[derive(Clone)] pub struct RocksDB { db: Arc diff --git a/crates/strategy/src/liquidator/liquidator.rs b/crates/strategy/src/liquidator/liquidator.rs index 74af662..d65ea8f 100644 --- a/crates/strategy/src/liquidator/liquidator.rs +++ b/crates/strategy/src/liquidator/liquidator.rs @@ -1,19 +1,31 @@ use crate::liquidator::db_writer::RocksDB; -use crate::liquidator::datasets::{ - aave_execute_borrow::process_aave_execute_borrow, +use crate::{ core::strategy::Strategy}; +use alloy_eips::NumHash; +use alloy_rpc_types::{ AccessList }; +use eyre::{Result, Error}; +use reth_revm::{ + state::Bytecode, +}; +use reth_primitives::{TransactionSigned, Receipt, RecoveredBlock, Block, Header}; +use alloy_primitives::{ Address}; +use reth_node_api::{ConfigureEvm, FullNodeComponents}; +use reth_tracing::tracing; +use reth_transaction_pool::PoolTransaction; +use reth_provider::{ BlockHashReader, DBProvider, LatestStateProviderRef, StateCommitmentProvider }; +use reth::primitives::{ NodePrimitives}; +use searcher_reth_manager::{ + common::{ CommonStrategyConfig, StrategyConfig }, + gas::GasConfig, + types::{StrategyCandidates} }; -use alloy_consensus::{Header}; -use eyre::Result; use reth::builder::NodeTypes; -use reth::primitives::NodePrimitives; -use reth_primitives::{TransactionSigned, Receipt, RecoveredBlock, Block}; -use reth_node_api::{ConfigureEvm, FullNodeComponents, FullNodeTypes}; -use reth_tracing::tracing::{info, warn}; use std::time::Instant; -use searcher_reth_manager::{ - common::{ CommonStrategyConfig, ONE_ETHER, StrategyConfig }, - gas::GasConfig +use crate::liquidator::db_writer::TableName; +use crate::liquidator::datasets::{ + dolomite_borrow_position::process_dolomite_borrow_positions, + aave_execute_borrow::process_aave_execute_borrow, }; +use crate::liquidator::LiquidatorTodoAction; // Structure to hold all the components needed for processing #[derive(Clone)] @@ -24,7 +36,7 @@ pub struct ProcessingComponents { struct ProcessorInfo { table_name: &'static str, - processor_name: &'static str, + processor_name: String, processor: for<'a> fn( &'a (RecoveredBlock, Vec), ProcessingComponents, @@ -35,7 +47,7 @@ struct ProcessorInfo { impl ProcessorInfo { fn new( table_name: &'static str, - processor_name: &'static str, + processor_name: &str, processor: for<'a> fn( &'a (RecoveredBlock, Vec), ProcessingComponents, @@ -44,27 +56,32 @@ impl ProcessorInfo { ) -> Self { Self { table_name, - processor_name, + processor_name: processor_name.to_string(), processor, } } } -pub struct Liquidator { +pub struct Liquidator { processors: Vec>, config: StrategyConfig, } -impl Strategy for Liquidator { - fn new(config: Config) -> Self { - let mut liquidator = Self { - processors: Vec::new(), - config, - }; +impl Strategy for Liquidator +where + Node::Types: NodeTypes, + ::Primitives: NodePrimitives< + BlockHeader = Header, + Block = Block, + Receipt = Receipt, + SignedTx = TransactionSigned, + >, +{ + // TODO : define action for liquidator + type Action = LiquidatorTodoAction; - liquidator.add_processor("aave_execute_borrow", "AaveExecuteBorrow"); - info!("Initialized liquidator with processors: {:?}", liquidator.list_processors()); - liquidator + fn new(config: StrategyConfig) -> Self { + Self { processors: Vec::new(), config: config.clone() } } fn gas_config(&self) -> GasConfig { @@ -86,7 +103,17 @@ impl Strategy for Liquidator { chain_id = chain_id, "Entered prepare()" ); - // TODO : define actions for prepare stage + let config_result = self.config.load_candidates(chain_id); + let processors = match config_result { + StrategyCandidates::Processors(c) => c, + _ => vec![], + }; + // processors for 문 돌면서 add_processor 해줘야함 + for processor in processors { + self.add_processor(processor.table, processor.processor); + } + + tracing::info!("Initialized liquidator with processors: {:?}", self.list_processors()); tracing::info!( target: "liquidator", event = "prepare_exit", @@ -109,7 +136,179 @@ impl Strategy for Liquidator { // 2. filter candidates which can be liquidated // 3. return the candidates // 4. store new candidates that indexer processed + Ok(None) } } +impl Liquidator +where + Node::Types: NodeTypes, + ::Primitives: NodePrimitives< + BlockHeader = Header, + Block = Block, + Receipt = Receipt, + SignedTx = TransactionSigned, + >, + Node::Provider: reth::providers::BlockReader> + + reth::providers::HeaderProvider
+ + reth::providers::ReceiptProvider + + reth::providers::TransactionsProvider, + Node::Evm: ConfigureEvm +{ + // add_processor adds indexer processor to liquidator + fn add_processor(&mut self, table_name: String, processor_name: String) { + let table = match TableName::from_str(&table_name) { + Some(t) => t, + None => return, // Skip if table name is not recognized + }; + + let processor = match table { + TableName::DolomiteBorrowPositions => ProcessorInfo::new( + table.as_str(), + &processor_name, + |block_data, components, db| Box::pin(process_dolomite_borrow_positions::(block_data, components, db)) + ), + TableName::AaveExecuteBorrow => ProcessorInfo::new( + table.as_str(), + &processor_name, + |block_data, components, db| Box::pin(process_aave_execute_borrow::(block_data, components, db)) + ), + }; + self.processors.push(processor); + } + + fn list_processors(&self) -> Vec<&str> { + self.processors.iter().map(|p| p.processor_name.as_str()).collect() + } + + async fn process_blocks( + &self, + blocks_and_receipts: impl Iterator, &Vec)>, + db: &RocksDB, + provider: Node::Provider, + ) -> Result<()> + where + Node::Evm: ConfigureEvm + { + // Convert the iterator items into owned values directly + let blocks_and_receipts: Vec<_> = blocks_and_receipts + .map(|(block, receipts)| (block.clone(), receipts.clone())) + .collect(); + + for (block, receipts) in blocks_and_receipts { + let block_number = block.number; + + // Create components for processing + let components = ProcessingComponents { + provider: provider.clone(), + config: self.config.clone(), + }; + + let block_data = (block, receipts); + if let Err(e) = self.process_block_data(&block_data, components, db).await { + tracing::warn!("Failed to process block {}: {}", block_number, e); + } + } + + Ok(()) + } + + async fn process_block_data( + &self, + block_data: &(RecoveredBlock, Vec), + components: ProcessingComponents, + db: &RocksDB + ) -> Result<()> + where + Node::Types: NodeTypes, + ::Primitives: NodePrimitives< + BlockHeader = Header, + Block = Block, + Receipt = Receipt, + SignedTx = TransactionSigned + >, + { + let block_number = block_data.0.number; + + // Create a vector to store all processing tasks + let mut tasks = Vec::new(); + + // Spawn a task for each enabled processor + for processor in &self.processors { + // Clone necessary data for the task + let processor_name = processor.processor_name.clone(); + let processor_fn = processor.processor; + let block_data = block_data.clone(); + let components = components.clone(); + let db = db.clone(); // Clone db before spawn + + // Spawn the task + let task = tokio::spawn(async move { + let event_start_time = Instant::now(); + match processor_fn(&block_data, components, &db).await { + Ok(()) => Ok((processor_name, event_start_time.elapsed())), + Err(e) => Err((processor_name, e.to_string())) + } + }); + + tasks.push(task); + } + + // Wait for all tasks to complete and collect results + let mut total_records = 0; + let mut event_results = Vec::new(); + let mut failed_events = Vec::new(); + + for task in tasks { + match task.await { + Ok(Ok((name, duration))) => { + total_records += 1; // Assuming each event writes one record for now + event_results.push((name, duration)); + } + Ok(Err((name, error))) => { + failed_events.push((name, error)); + } + Err(e) => { + tracing::warn!("Task join error: {}", e); + } + } + } + + // Sort events by name for consistent logging + event_results.sort_by(|a, b| a.0.cmp(&b.0)); + + // Create a consolidated success log + if !event_results.is_empty() { + let events_summary: Vec = event_results + .iter() + .map(|(name, time)| { + format!("{}({}, {:.2}s)", name, 1, time.as_secs_f64()) + }) + .collect(); + + tracing::info!( + "exex{{id=\"exex-indexer\"}}: Block {} processed - Events: [{}], Total records: {}", + block_number, + events_summary.join(", "), + total_records, + ); + } + + // Create a consolidated error log + if !failed_events.is_empty() { + let failure_summary: Vec = failed_events + .iter() + .map(|(name, error)| format!("{}: {}", name, error)) + .collect(); + + tracing::warn!( + "exex{{id=\"exex-indexer\"}}: Block {} failures - {}", + block_number, + failure_summary.join(", ") + ); + } + + Ok(()) + } +} \ No newline at end of file diff --git a/crates/strategy/src/liquidator/mod.rs b/crates/strategy/src/liquidator/mod.rs new file mode 100644 index 0000000..6813e6d --- /dev/null +++ b/crates/strategy/src/liquidator/mod.rs @@ -0,0 +1,9 @@ +mod liquidator; +mod db_writer; +mod datasets; +mod types; + +pub use liquidator::*; +pub use db_writer::*; +pub use datasets::*; +pub use types::*; diff --git a/crates/strategy/src/liquidator/types.rs b/crates/strategy/src/liquidator/types.rs new file mode 100644 index 0000000..137d44f --- /dev/null +++ b/crates/strategy/src/liquidator/types.rs @@ -0,0 +1,10 @@ +use alloy_sol_types::sol; + +sol! { + #[derive(Debug)] + struct LiquidatorTodoAction { + uint256 amount; + address token; + address to; + } +} \ No newline at end of file diff --git a/crates/strategy/src/path_finding/path_finder.rs b/crates/strategy/src/path_finding/path_finder.rs index 16f5535..88be320 100644 --- a/crates/strategy/src/path_finding/path_finder.rs +++ b/crates/strategy/src/path_finding/path_finder.rs @@ -32,7 +32,7 @@ use reth_transaction_pool::PoolTransaction; use searcher_reth_manager::{ common::{CommonStrategyConfig, ONE_ETHER, StrategyConfig}, gas::GasConfig, - types::CandidateEntry, + types::{CandidateEntry, StrategyCandidates}, }; use crate::path_finding::types::executeCall; @@ -81,7 +81,11 @@ impl Strategy for PathFinder { chain_id = chain_id, "Entered prepare()" ); - let candidates = self.config.load_candidates(chain_id); + let config_result = self.config.load_candidates(chain_id); + let candidates = match config_result { + StrategyCandidates::Candidates(c) => c, + _ => vec![], + }; tracing::info!( target: "path-finder", event = "candidates_count", From 73fa08fb1570482d09edf7c9e3fc2b2ad01769b9 Mon Sep 17 00:00:00 2001 From: 0xwiederholen <0xwiederholen@gmail.com> Date: Fri, 15 Aug 2025 22:59:29 +0900 Subject: [PATCH 09/12] feat: resolve trait bound conflict --- bin/searcher-reth/src/main.rs | 4 +- crates/extension/src/exex.rs | 8 +- .../datasets/aave_execute_borrow.rs | 6 +- .../datasets/dolomite_borrow_position.rs | 6 +- crates/strategy/src/liquidator/db_writer.rs | 5 +- crates/strategy/src/liquidator/liquidator.rs | 76 +++---------------- 6 files changed, 21 insertions(+), 84 deletions(-) diff --git a/bin/searcher-reth/src/main.rs b/bin/searcher-reth/src/main.rs index d7a2836..90c0b12 100644 --- a/bin/searcher-reth/src/main.rs +++ b/bin/searcher-reth/src/main.rs @@ -34,8 +34,8 @@ fn main() -> eyre::Result<()> { }); // Install Exex for various strategies - let searcher_exex = SearcherExEx::new(wallet, signal_manager.subscribe()); - let liquidator_exex = LiquidatorExEx::new(wallet, signal_manager.subscribe()); + let searcher_exex = SearcherExEx::new(wallet.clone(), signal_manager.subscribe()); + let liquidator_exex = LiquidatorExEx::new(wallet.clone(), signal_manager.subscribe()); let mut node_builder = builder.node(EthereumNode::default()); diff --git a/crates/extension/src/exex.rs b/crates/extension/src/exex.rs index 59b9c72..2a2ffd9 100644 --- a/crates/extension/src/exex.rs +++ b/crates/extension/src/exex.rs @@ -1,24 +1,20 @@ use std::{future::Future, sync::Arc}; -use eyre::{Result, WrapErr}; -use futures_util::{StreamExt, TryStreamExt}; +use eyre::{Result}; +use futures_util::{StreamExt}; use reth::network::NetworkInfo; -use reth::builder::NodeTypes; use reth_exex::{ ExExContext, ExExEvent, ExExNotification }; use reth_node_api::{ FullNodeComponents, FullNodeTypes }; use reth_provider::{ AccountReader, BlockHashReader, BlockReaderIdExt, ChainSpecProvider, DatabaseProviderFactory, LatestStateProviderRef, ReceiptProvider, StateCommitmentProvider, }; -use reth_primitives::{TransactionSigned, Receipt}; use reth_tracing::tracing::{ self }; use reth_transaction_pool::{ EthPooledTransaction, TransactionPool }; use searcher_reth_manager::SignalType; use searcher_reth_strategy::core::strategy::Strategy; use tokio::sync::broadcast; use crate::relayer_pool::{ RelayerMessage, RelayerPool, WalletPool }; -use alloy_consensus::{Header, Block}; -use reth::primitives::{NodePrimitives}; pub struct LiquidatorExEx { pub wallet: Arc, diff --git a/crates/strategy/src/liquidator/datasets/aave_execute_borrow.rs b/crates/strategy/src/liquidator/datasets/aave_execute_borrow.rs index 60a5890..feb7a6b 100644 --- a/crates/strategy/src/liquidator/datasets/aave_execute_borrow.rs +++ b/crates/strategy/src/liquidator/datasets/aave_execute_borrow.rs @@ -3,9 +3,8 @@ use alloy_sol_types::{sol, SolEvent}; use alloy_primitives::{address, Address}; use reth_primitives::{RecoveredBlock, Receipt, Block}; use eyre::Result; -use reth_node_api::FullNodeComponents; use reth_tracing::tracing::debug; -use crate::liquidator::ProcessingComponents; + // Pool contarct address const ARBITRUM_AAVE_CONTRACT_ADDRESS: Address = address!("0x794a61358D6845594F94dc1DB02A252b5b4814aD"); @@ -21,9 +20,8 @@ sol! { ); } -pub async fn process_aave_execute_borrow( +pub async fn process_aave_execute_borrow( block_data: &(RecoveredBlock, Vec), - components: ProcessingComponents, db: &RocksDB, ) -> Result<()> { let block = &block_data.0; diff --git a/crates/strategy/src/liquidator/datasets/dolomite_borrow_position.rs b/crates/strategy/src/liquidator/datasets/dolomite_borrow_position.rs index bcb5e17..1cddad3 100644 --- a/crates/strategy/src/liquidator/datasets/dolomite_borrow_position.rs +++ b/crates/strategy/src/liquidator/datasets/dolomite_borrow_position.rs @@ -3,9 +3,8 @@ use alloy_sol_types::{sol, SolEvent}; use alloy_primitives::{address, Address}; use reth_primitives::{RecoveredBlock, Receipt, Block}; use eyre::Result; -use reth_node_api::FullNodeComponents; use reth_tracing::tracing::debug; -use crate::liquidator::ProcessingComponents; + // BorrowPositionProxyV2 address const BERA_DOLOMITE_CONTRACT_ADDRESS: Address = address!("0xC06271eb97d960F4034DDF953e16271CcB2B10BD"); @@ -16,9 +15,8 @@ sol! { ); } -pub async fn process_dolomite_borrow_positions( +pub async fn process_dolomite_borrow_positions( block_data: &(RecoveredBlock, Vec), - components: ProcessingComponents, db: &RocksDB, ) -> Result<()> { let block = &block_data.0; diff --git a/crates/strategy/src/liquidator/db_writer.rs b/crates/strategy/src/liquidator/db_writer.rs index 45d6916..b94bd3c 100644 --- a/crates/strategy/src/liquidator/db_writer.rs +++ b/crates/strategy/src/liquidator/db_writer.rs @@ -1,7 +1,6 @@ use std::sync::Arc; -use alloy_primitives::{Address, FixedBytes, Uint, Signed}; -use std::collections::{HashSet, HashMap}; -use rocksdb::{DB, ColumnFamily, Options, ColumnFamilyDescriptor}; +use std::collections::HashSet; +use rocksdb::{DB, Options, ColumnFamilyDescriptor}; // database table 이름 타입으로 정의 pub enum TableName { diff --git a/crates/strategy/src/liquidator/liquidator.rs b/crates/strategy/src/liquidator/liquidator.rs index d65ea8f..f4c235a 100644 --- a/crates/strategy/src/liquidator/liquidator.rs +++ b/crates/strategy/src/liquidator/liquidator.rs @@ -6,19 +6,16 @@ use eyre::{Result, Error}; use reth_revm::{ state::Bytecode, }; -use reth_primitives::{TransactionSigned, Receipt, RecoveredBlock, Block, Header}; +use reth_primitives::{Receipt, RecoveredBlock, Block}; use alloy_primitives::{ Address}; -use reth_node_api::{ConfigureEvm, FullNodeComponents}; use reth_tracing::tracing; use reth_transaction_pool::PoolTransaction; use reth_provider::{ BlockHashReader, DBProvider, LatestStateProviderRef, StateCommitmentProvider }; -use reth::primitives::{ NodePrimitives}; use searcher_reth_manager::{ common::{ CommonStrategyConfig, StrategyConfig }, gas::GasConfig, types::{StrategyCandidates} }; -use reth::builder::NodeTypes; use std::time::Instant; use crate::liquidator::db_writer::TableName; use crate::liquidator::datasets::{ @@ -27,30 +24,21 @@ use crate::liquidator::datasets::{ }; use crate::liquidator::LiquidatorTodoAction; -// Structure to hold all the components needed for processing -#[derive(Clone)] -pub struct ProcessingComponents { - pub provider: Node::Provider, - config: StrategyConfig, -} - -struct ProcessorInfo { +struct ProcessorInfo { table_name: &'static str, processor_name: String, processor: for<'a> fn( &'a (RecoveredBlock, Vec), - ProcessingComponents, &'a RocksDB ) -> futures::future::BoxFuture<'a, Result<()>>, } -impl ProcessorInfo { +impl ProcessorInfo { fn new( table_name: &'static str, processor_name: &str, processor: for<'a> fn( &'a (RecoveredBlock, Vec), - ProcessingComponents, &'a RocksDB ) -> futures::future::BoxFuture<'a, Result<()>>, ) -> Self { @@ -62,21 +50,12 @@ impl ProcessorInfo { } } -pub struct Liquidator { - processors: Vec>, +pub struct Liquidator{ + processors: Vec, config: StrategyConfig, } -impl Strategy for Liquidator -where - Node::Types: NodeTypes, - ::Primitives: NodePrimitives< - BlockHeader = Header, - Block = Block, - Receipt = Receipt, - SignedTx = TransactionSigned, - >, -{ +impl Strategy for Liquidator { // TODO : define action for liquidator type Action = LiquidatorTodoAction; @@ -141,21 +120,7 @@ where } -impl Liquidator -where - Node::Types: NodeTypes, - ::Primitives: NodePrimitives< - BlockHeader = Header, - Block = Block, - Receipt = Receipt, - SignedTx = TransactionSigned, - >, - Node::Provider: reth::providers::BlockReader> - + reth::providers::HeaderProvider
- + reth::providers::ReceiptProvider - + reth::providers::TransactionsProvider, - Node::Evm: ConfigureEvm -{ +impl Liquidator { // add_processor adds indexer processor to liquidator fn add_processor(&mut self, table_name: String, processor_name: String) { let table = match TableName::from_str(&table_name) { @@ -167,12 +132,12 @@ where TableName::DolomiteBorrowPositions => ProcessorInfo::new( table.as_str(), &processor_name, - |block_data, components, db| Box::pin(process_dolomite_borrow_positions::(block_data, components, db)) + |block_data, db| Box::pin(process_dolomite_borrow_positions(block_data, db)) ), TableName::AaveExecuteBorrow => ProcessorInfo::new( table.as_str(), &processor_name, - |block_data, components, db| Box::pin(process_aave_execute_borrow::(block_data, components, db)) + |block_data, db| Box::pin(process_aave_execute_borrow(block_data, db)) ), }; self.processors.push(processor); @@ -186,10 +151,7 @@ where &self, blocks_and_receipts: impl Iterator, &Vec)>, db: &RocksDB, - provider: Node::Provider, ) -> Result<()> - where - Node::Evm: ConfigureEvm { // Convert the iterator items into owned values directly let blocks_and_receipts: Vec<_> = blocks_and_receipts @@ -198,15 +160,8 @@ where for (block, receipts) in blocks_and_receipts { let block_number = block.number; - - // Create components for processing - let components = ProcessingComponents { - provider: provider.clone(), - config: self.config.clone(), - }; - let block_data = (block, receipts); - if let Err(e) = self.process_block_data(&block_data, components, db).await { + if let Err(e) = self.process_block_data(&block_data, db).await { tracing::warn!("Failed to process block {}: {}", block_number, e); } } @@ -217,17 +172,9 @@ where async fn process_block_data( &self, block_data: &(RecoveredBlock, Vec), - components: ProcessingComponents, db: &RocksDB ) -> Result<()> where - Node::Types: NodeTypes, - ::Primitives: NodePrimitives< - BlockHeader = Header, - Block = Block, - Receipt = Receipt, - SignedTx = TransactionSigned - >, { let block_number = block_data.0.number; @@ -240,13 +187,12 @@ where let processor_name = processor.processor_name.clone(); let processor_fn = processor.processor; let block_data = block_data.clone(); - let components = components.clone(); let db = db.clone(); // Clone db before spawn // Spawn the task let task = tokio::spawn(async move { let event_start_time = Instant::now(); - match processor_fn(&block_data, components, &db).await { + match processor_fn(&block_data, &db).await { Ok(()) => Ok((processor_name, event_start_time.elapsed())), Err(e) => Err((processor_name, e.to_string())) } From fff62e2686a12a8498522e76cbf68822e1c95920 Mon Sep 17 00:00:00 2001 From: suha jin <89185836+djm07073@users.noreply.github.com> Date: Fri, 15 Aug 2025 23:41:20 +0900 Subject: [PATCH 10/12] feat: lint ci --- .github/workflows/rust-ci.yml | 58 +++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 .github/workflows/rust-ci.yml diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml new file mode 100644 index 0000000..ffe52a6 --- /dev/null +++ b/.github/workflows/rust-ci.yml @@ -0,0 +1,58 @@ +name: Test & Lint CI + +on: + push: + branches: [ main, develop ] + pull_request: + branches: [ main, develop ] + +env: + CARGO_TERM_COLOR: always + +jobs: + check: + name: Rust CI Checks + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo- + + - name: Check formatting + run: | + cargo fmt --all -- --check + if [ $? -ne 0 ]; then + echo "::error::Code is not formatted. Please run 'cargo fmt' locally." + exit 1 + fi + + - name: Run clippy + run: | + cargo clippy --all --all-targets -- -D warnings + if [ $? -ne 0 ]; then + echo "::error::Clippy found issues. Please fix them locally." + exit 1 + fi + + - name: Run tests + run: | + cargo test --all + if [ $? -ne 0 ]; then + echo "::error::Tests failed. Please fix them locally." + exit 1 + fi \ No newline at end of file From 9bade8617ef9a8365bd30bbde4e525503883dbd9 Mon Sep 17 00:00:00 2001 From: suha jin <89185836+djm07073@users.noreply.github.com> Date: Wed, 20 Aug 2025 12:00:37 +0900 Subject: [PATCH 11/12] fix(liquidator): liquidator contract type --- bin/searcher-reth/src/main.rs | 15 ++- crates/extension/src/exex.rs | 39 +++---- crates/extension/src/lib.rs | 2 +- crates/manager/src/common.rs | 27 ++--- crates/manager/src/strategy/liquidator/mod.rs | 11 +- crates/manager/src/strategy/mod.rs | 2 +- crates/strategy/src/lib.rs | 2 +- .../datasets/aave_execute_borrow.rs | 17 +-- .../datasets/dolomite_borrow_position.rs | 15 +-- .../strategy/src/liquidator/datasets/mod.rs | 2 +- crates/strategy/src/liquidator/db_writer.rs | 21 ++-- crates/strategy/src/liquidator/liquidator.rs | 109 ++++++++---------- crates/strategy/src/liquidator/mod.rs | 8 +- crates/strategy/src/liquidator/types.rs | 32 ++++- 14 files changed, 157 insertions(+), 145 deletions(-) diff --git a/bin/searcher-reth/src/main.rs b/bin/searcher-reth/src/main.rs index 90c0b12..d332b72 100644 --- a/bin/searcher-reth/src/main.rs +++ b/bin/searcher-reth/src/main.rs @@ -5,14 +5,18 @@ use reth::chainspec::EthereumChainSpecParser; use reth_node_ethereum::EthereumNode; use reth_tracing::tracing::error; use searcher_reth_extension::{ - exex::{SearcherExEx, LiquidatorExEx}, + exex::{LiquidatorExEx, SearcherExEx}, relayer_pool::WalletPool, strategy::{ - core::strategy::Strategy, path_finding::PathFinder, profit_reporter::init_reporter, liquidator::Liquidator, + core::strategy::Strategy, liquidator::Liquidator, path_finding::PathFinder, + profit_reporter::init_reporter, }, util::signal_manager::SignalManager, }; -use searcher_reth_manager::{common::{LIQUIDATOR_EXEX_ID, PATH_FINDER_EXEX_ID}, manager::ConfigManager}; +use searcher_reth_manager::{ + common::{LIQUIDATOR_EXEX_ID, PATH_FINDER_EXEX_ID}, + manager::ConfigManager, +}; fn main() -> eyre::Result<()> { let config = Arc::new(RwLock::new(ConfigManager::from_file("env.toml")?)); @@ -36,7 +40,7 @@ fn main() -> eyre::Result<()> { // Install Exex for various strategies let searcher_exex = SearcherExEx::new(wallet.clone(), signal_manager.subscribe()); let liquidator_exex = LiquidatorExEx::new(wallet.clone(), signal_manager.subscribe()); - + let mut node_builder = builder.node(EthereumNode::default()); // Install PathFinder strategy @@ -49,7 +53,8 @@ fn main() -> eyre::Result<()> { // Install Indexer node_builder = node_builder.install_exex(LIQUIDATOR_EXEX_ID, { - let liquidator_cfg = config.clone().read().unwrap().get_strategy(LIQUIDATOR_EXEX_ID).unwrap(); + let liquidator_cfg = + config.clone().read().unwrap().get_strategy(LIQUIDATOR_EXEX_ID).unwrap(); let liquidator = Liquidator::new(liquidator_cfg); move |ctx| liquidator_exex.run(ctx, liquidator) }); diff --git a/crates/extension/src/exex.rs b/crates/extension/src/exex.rs index 2a2ffd9..3ecd8bf 100644 --- a/crates/extension/src/exex.rs +++ b/crates/extension/src/exex.rs @@ -1,20 +1,20 @@ use std::{future::Future, sync::Arc}; -use eyre::{Result}; -use futures_util::{StreamExt}; +use crate::relayer_pool::{RelayerMessage, RelayerPool, WalletPool}; +use eyre::Result; +use futures_util::StreamExt; use reth::network::NetworkInfo; -use reth_exex::{ ExExContext, ExExEvent, ExExNotification }; -use reth_node_api::{ FullNodeComponents, FullNodeTypes }; +use reth_exex::{ExExContext, ExExEvent, ExExNotification}; +use reth_node_api::{FullNodeComponents, FullNodeTypes}; use reth_provider::{ AccountReader, BlockHashReader, BlockReaderIdExt, ChainSpecProvider, DatabaseProviderFactory, LatestStateProviderRef, ReceiptProvider, StateCommitmentProvider, }; -use reth_tracing::tracing::{ self }; -use reth_transaction_pool::{ EthPooledTransaction, TransactionPool }; +use reth_tracing::tracing::{self}; +use reth_transaction_pool::{EthPooledTransaction, TransactionPool}; use searcher_reth_manager::SignalType; use searcher_reth_strategy::core::strategy::Strategy; use tokio::sync::broadcast; -use crate::relayer_pool::{ RelayerMessage, RelayerPool, WalletPool }; pub struct LiquidatorExEx { pub wallet: Arc, @@ -29,27 +29,23 @@ impl LiquidatorExEx { pub async fn run( self, mut ctx: ExExContext, - mut strategy: S + mut strategy: S, ) -> Result>> where S: Strategy, Node: FullNodeComponents, Node::Pool: TransactionPool, - <::Provider as DatabaseProviderFactory>::Provider: BlockHashReader + - StateCommitmentProvider, - Node::Provider: BlockReaderIdExt + ReceiptProvider + AccountReader + ChainSpecProvider + <::Provider as DatabaseProviderFactory>::Provider: + BlockHashReader + StateCommitmentProvider, + Node::Provider: BlockReaderIdExt + ReceiptProvider + AccountReader + ChainSpecProvider, { let wallet = self.wallet.clone(); let signal_rx = self.signal_rx.resubscribe(); let vault = strategy.get_vault(); Ok(async move { let relayer_pool = Arc::new( - RelayerPool::new( - ctx.components.clone(), - wallet, - signal_rx, - strategy.gas_config() - ).await? + RelayerPool::new(ctx.components.clone(), wallet, signal_rx, strategy.gas_config()) + .await?, ); let relayer_tx = relayer_pool.start().await?; tracing::info!( @@ -74,7 +70,7 @@ impl LiquidatorExEx { let profitable_candidates = strategy.find_profitable_candidates( num_hash, latest_state_provider, - Vec::::new() + Vec::::new(), )?; tracing::info!( @@ -98,16 +94,14 @@ impl LiquidatorExEx { let r = relayer_channel.send(message).await; match r { - Ok(_) => - tracing::info!( + Ok(_) => tracing::info!( target: "reth-exex", event = "send_candidates_to_relayer_pool", success = true, num_hash = ?num_hash, "Successfully sent candidates to relayer pool" ), - Err(e) => - tracing::error!( + Err(e) => tracing::error!( target: "reth-exex", event = "send_candidates_to_relayer_pool", success = false, @@ -123,7 +117,6 @@ impl LiquidatorExEx { Ok(()) }) } - } pub struct SearcherExEx { diff --git a/crates/extension/src/lib.rs b/crates/extension/src/lib.rs index 90e0f10..0b94732 100644 --- a/crates/extension/src/lib.rs +++ b/crates/extension/src/lib.rs @@ -7,4 +7,4 @@ pub mod util { pub mod strategy { pub use searcher_reth_strategy::*; -} \ No newline at end of file +} diff --git a/crates/manager/src/common.rs b/crates/manager/src/common.rs index 7c2eb83..57f58b9 100644 --- a/crates/manager/src/common.rs +++ b/crates/manager/src/common.rs @@ -5,8 +5,8 @@ use serde::{Deserialize, Serialize}; use crate::{ gas::GasConfig, - strategy::{path_finder::PathFinderConfig, liquidator::LiquidatorConfig}, - types::StrategyCandidates + strategy::{liquidator::LiquidatorConfig, path_finder::PathFinderConfig}, + types::StrategyCandidates, }; // Strategy configuration @@ -18,7 +18,7 @@ pub const LIQUIDATOR_EXEX_ID: &str = "liquidator"; pub enum StrategyConfig { #[serde(rename = "path-finder")] PathFinder(PathFinderConfig), - #[serde(rename = "liquidator")] + #[serde(rename = "liquidator")] Liquidator(LiquidatorConfig), // #[serde(rename = "arbitrage")] Arbitrage(ArbitrageConfig), } @@ -74,8 +74,7 @@ impl CommonStrategyConfig for StrategyConfig { tracing::warn!("Invalid vault address, using default ZERO address"); Address::ZERO } - } - // TODO: Add other strategy configurations + }, // TODO: Add other strategy configurations } } @@ -100,7 +99,7 @@ impl CommonStrategyConfig for StrategyConfig { Bytecode::default() // Return an empty bytecode on error } } - }, + } StrategyConfig::Liquidator(config) => { tracing::info!("Loading strategy contract for Liquidator: {:?}", config.contract); let bytes = match hex::decode(config.contract.clone()) { @@ -120,8 +119,7 @@ impl CommonStrategyConfig for StrategyConfig { Bytecode::default() // Return an empty bytecode on error } } - } - // TODO: Add other strategy configurations + } // TODO: Add other strategy configurations } } @@ -134,7 +132,7 @@ impl CommonStrategyConfig for StrategyConfig { U256::from((max_profit * (ONE_ETHER as f64)) as u128), U256::from((min_profit * (ONE_ETHER as f64)) as u128), ) - }, + } StrategyConfig::Liquidator(config) => { let max_profit = config.max_profit.parse::().unwrap(); let min_profit = config.min_profit.parse::().unwrap(); @@ -142,8 +140,7 @@ impl CommonStrategyConfig for StrategyConfig { U256::from((max_profit * (ONE_ETHER as f64)) as u128), U256::from((min_profit * (ONE_ETHER as f64)) as u128), ) - } - // TODO: Add other strategy configurations + } // TODO: Add other strategy configurations } } @@ -171,11 +168,15 @@ impl CommonStrategyConfig for StrategyConfig { StrategyCandidates::Candidates(vec![]) // Return an empty vector on error } } - }, + } StrategyConfig::Liquidator(config) => { match config.load_processors(chain_id) { Ok(processors) => { - tracing::info!("Loaded {} processors for chain_id: {}", processors.len(), chain_id); + tracing::info!( + "Loaded {} processors for chain_id: {}", + processors.len(), + chain_id + ); StrategyCandidates::Processors(processors) } Err(e) => { diff --git a/crates/manager/src/strategy/liquidator/mod.rs b/crates/manager/src/strategy/liquidator/mod.rs index cf9475a..298a2d3 100644 --- a/crates/manager/src/strategy/liquidator/mod.rs +++ b/crates/manager/src/strategy/liquidator/mod.rs @@ -1,8 +1,7 @@ -use std::{collections::HashMap, fs::File, io::BufReader, path::PathBuf}; +use crate::{gas::GasConfig, types::ProcessorEntry}; use eyre::eyre; use serde::{Deserialize, Serialize}; -use crate::types::ProcessorEntry; -use crate::gas::GasConfig; +use std::{collections::HashMap, fs::File, io::BufReader, path::PathBuf}; #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] @@ -51,11 +50,11 @@ impl LiquidatorConfig { let processors: HashMap> = serde_json::from_reader(BufReader::new(file)) .map_err(|e| eyre!("Failed to parse routes JSON: {}", e))?; - + let chain_processors = processors .get(&chain_id.to_string()) .ok_or_else(|| eyre!("No processors found for chain_id: {}", chain_id))?; - + Ok(chain_processors.clone()) } -} \ No newline at end of file +} diff --git a/crates/manager/src/strategy/mod.rs b/crates/manager/src/strategy/mod.rs index 015e80a..256a179 100644 --- a/crates/manager/src/strategy/mod.rs +++ b/crates/manager/src/strategy/mod.rs @@ -1,2 +1,2 @@ +pub mod liquidator; pub mod path_finder; -pub mod liquidator; \ No newline at end of file diff --git a/crates/strategy/src/lib.rs b/crates/strategy/src/lib.rs index 8fbf17e..161dd82 100644 --- a/crates/strategy/src/lib.rs +++ b/crates/strategy/src/lib.rs @@ -1,4 +1,4 @@ pub mod core; -pub mod path_finding; pub mod liquidator; +pub mod path_finding; pub mod profit_reporter; diff --git a/crates/strategy/src/liquidator/datasets/aave_execute_borrow.rs b/crates/strategy/src/liquidator/datasets/aave_execute_borrow.rs index feb7a6b..10e12c6 100644 --- a/crates/strategy/src/liquidator/datasets/aave_execute_borrow.rs +++ b/crates/strategy/src/liquidator/datasets/aave_execute_borrow.rs @@ -1,12 +1,13 @@ use crate::liquidator::db_writer::RocksDB; -use alloy_sol_types::{sol, SolEvent}; -use alloy_primitives::{address, Address}; -use reth_primitives::{RecoveredBlock, Receipt, Block}; +use alloy_primitives::{Address, address}; +use alloy_sol_types::{SolEvent, sol}; use eyre::Result; +use reth_primitives::{Block, Receipt, RecoveredBlock}; use reth_tracing::tracing::debug; // Pool contarct address -const ARBITRUM_AAVE_CONTRACT_ADDRESS: Address = address!("0x794a61358D6845594F94dc1DB02A252b5b4814aD"); +const ARBITRUM_AAVE_CONTRACT_ADDRESS: Address = + address!("0x794a61358D6845594F94dc1DB02A252b5b4814aD"); sol! { event Borrow( @@ -28,7 +29,8 @@ pub async fn process_aave_execute_borrow( let receipts = &block_data.1; // Iterate through transactions and their receipts - for (tx_idx, (tx, receipt)) in block.body().transactions.iter().zip(receipts.iter()).enumerate() { + for (tx_idx, (tx, receipt)) in block.body().transactions.iter().zip(receipts.iter()).enumerate() + { // Process each log in the receipt for (log_idx, log) in receipt.logs.iter().enumerate() { // Filter on dolomite contract address @@ -39,11 +41,11 @@ pub async fn process_aave_execute_borrow( } match Borrow::decode_raw_log(log.topics(), &log.data.data) { - Ok(create) => { + Ok(create) => { db.save( "aave_borrow", &format!("{}", create.onBehalfOf), - &format!("{}_{}", create.reserve, create.amount) + &format!("{}_{}", create.reserve, create.amount), ); } Err(e) => { @@ -55,4 +57,3 @@ pub async fn process_aave_execute_borrow( } Ok(()) } - diff --git a/crates/strategy/src/liquidator/datasets/dolomite_borrow_position.rs b/crates/strategy/src/liquidator/datasets/dolomite_borrow_position.rs index 1cddad3..7094fe1 100644 --- a/crates/strategy/src/liquidator/datasets/dolomite_borrow_position.rs +++ b/crates/strategy/src/liquidator/datasets/dolomite_borrow_position.rs @@ -1,12 +1,13 @@ use crate::liquidator::db_writer::RocksDB; -use alloy_sol_types::{sol, SolEvent}; -use alloy_primitives::{address, Address}; -use reth_primitives::{RecoveredBlock, Receipt, Block}; +use alloy_primitives::{Address, address}; +use alloy_sol_types::{SolEvent, sol}; use eyre::Result; +use reth_primitives::{Block, Receipt, RecoveredBlock}; use reth_tracing::tracing::debug; // BorrowPositionProxyV2 address -const BERA_DOLOMITE_CONTRACT_ADDRESS: Address = address!("0xC06271eb97d960F4034DDF953e16271CcB2B10BD"); +const BERA_DOLOMITE_CONTRACT_ADDRESS: Address = + address!("0xC06271eb97d960F4034DDF953e16271CcB2B10BD"); sol! { event BorrowPositionOpen( @@ -23,7 +24,8 @@ pub async fn process_dolomite_borrow_positions( let receipts = &block_data.1; // Iterate through transactions and their receipts - for (tx_idx, (tx, receipt)) in block.body().transactions.iter().zip(receipts.iter()).enumerate() { + for (tx_idx, (tx, receipt)) in block.body().transactions.iter().zip(receipts.iter()).enumerate() + { // Process each log in the receipt for (log_idx, log) in receipt.logs.iter().enumerate() { // Filter on dolomite contract address @@ -38,7 +40,7 @@ pub async fn process_dolomite_borrow_positions( db.save( "dolomite_borrow_positions", &format!("{}", create._borrowAccountNumber), - &format!("{}", create._borrower) + &format!("{}", create._borrower), ); } Err(e) => { @@ -50,4 +52,3 @@ pub async fn process_dolomite_borrow_positions( } Ok(()) } - diff --git a/crates/strategy/src/liquidator/datasets/mod.rs b/crates/strategy/src/liquidator/datasets/mod.rs index 7d0b52f..635209d 100644 --- a/crates/strategy/src/liquidator/datasets/mod.rs +++ b/crates/strategy/src/liquidator/datasets/mod.rs @@ -1,2 +1,2 @@ pub mod aave_execute_borrow; -pub mod dolomite_borrow_position; \ No newline at end of file +pub mod dolomite_borrow_position; diff --git a/crates/strategy/src/liquidator/db_writer.rs b/crates/strategy/src/liquidator/db_writer.rs index b94bd3c..03ac864 100644 --- a/crates/strategy/src/liquidator/db_writer.rs +++ b/crates/strategy/src/liquidator/db_writer.rs @@ -1,8 +1,7 @@ -use std::sync::Arc; -use std::collections::HashSet; -use rocksdb::{DB, Options, ColumnFamilyDescriptor}; +use rocksdb::{ColumnFamilyDescriptor, DB, Options}; +use std::{collections::HashSet, sync::Arc}; -// database table 이름 타입으로 정의 +// database table 이름 타입으로 정의 pub enum TableName { DolomiteBorrowPositions, AaveExecuteBorrow, @@ -13,7 +12,7 @@ impl TableName { match s { "dolomite_borrow_positions" => Some(TableName::DolomiteBorrowPositions), "aave_execute_borrow" => Some(TableName::AaveExecuteBorrow), - _ => None + _ => None, } } @@ -27,7 +26,7 @@ impl TableName { #[derive(Clone)] pub struct RocksDB { - db: Arc + db: Arc, } impl RocksDB { @@ -51,9 +50,7 @@ impl RocksDB { } } - RocksDB { - db: Arc::new(db) - } + RocksDB { db: Arc::new(db) } } pub fn save(&self, cf: &str, k: &str, v: &str) -> bool { @@ -68,11 +65,11 @@ impl RocksDB { let result = String::from_utf8(v).unwrap(); println!("Finding '{}' returns '{}'", k, result); Some(result) - }, + } Ok(None) => { println!("Finding '{}' returns None", k); None - }, + } Err(e) => { println!("Error retrieving value for {}: {}", k, e); None @@ -84,4 +81,4 @@ impl RocksDB { let cf = self.db.cf_handle(cf).expect("missing CF"); self.db.delete_cf(cf, k.as_bytes()).is_ok() } -} \ No newline at end of file +} diff --git a/crates/strategy/src/liquidator/liquidator.rs b/crates/strategy/src/liquidator/liquidator.rs index f4c235a..8e4421c 100644 --- a/crates/strategy/src/liquidator/liquidator.rs +++ b/crates/strategy/src/liquidator/liquidator.rs @@ -1,35 +1,36 @@ -use crate::liquidator::db_writer::RocksDB; -use crate::{ core::strategy::Strategy}; -use alloy_eips::NumHash; -use alloy_rpc_types::{ AccessList }; -use eyre::{Result, Error}; -use reth_revm::{ - state::Bytecode, +use crate::{ + core::strategy::Strategy, + liquidator::{ + LiquidationPayload, + datasets::{ + aave_execute_borrow::process_aave_execute_borrow, + dolomite_borrow_position::process_dolomite_borrow_positions, + }, + db_writer::{RocksDB, TableName}, + }, }; -use reth_primitives::{Receipt, RecoveredBlock, Block}; -use alloy_primitives::{ Address}; +use alloy_eips::NumHash; +use alloy_primitives::Address; +use alloy_rpc_types::AccessList; +use eyre::{Error, Result}; +use reth_primitives::{Block, Receipt, RecoveredBlock}; +use reth_provider::{BlockHashReader, DBProvider, LatestStateProviderRef, StateCommitmentProvider}; +use reth_revm::state::Bytecode; use reth_tracing::tracing; use reth_transaction_pool::PoolTransaction; -use reth_provider::{ BlockHashReader, DBProvider, LatestStateProviderRef, StateCommitmentProvider }; use searcher_reth_manager::{ - common::{ CommonStrategyConfig, StrategyConfig }, + common::{CommonStrategyConfig, StrategyConfig}, gas::GasConfig, - types::{StrategyCandidates} + types::StrategyCandidates, }; use std::time::Instant; -use crate::liquidator::db_writer::TableName; -use crate::liquidator::datasets::{ - dolomite_borrow_position::process_dolomite_borrow_positions, - aave_execute_borrow::process_aave_execute_borrow, -}; -use crate::liquidator::LiquidatorTodoAction; struct ProcessorInfo { table_name: &'static str, processor_name: String, processor: for<'a> fn( &'a (RecoveredBlock, Vec), - &'a RocksDB + &'a RocksDB, ) -> futures::future::BoxFuture<'a, Result<()>>, } @@ -39,25 +40,21 @@ impl ProcessorInfo { processor_name: &str, processor: for<'a> fn( &'a (RecoveredBlock, Vec), - &'a RocksDB + &'a RocksDB, ) -> futures::future::BoxFuture<'a, Result<()>>, ) -> Self { - Self { - table_name, - processor_name: processor_name.to_string(), - processor, - } + Self { table_name, processor_name: processor_name.to_string(), processor } } } -pub struct Liquidator{ +pub struct Liquidator { processors: Vec, config: StrategyConfig, } impl Strategy for Liquidator { // TODO : define action for liquidator - type Action = LiquidatorTodoAction; + type Action = LiquidationPayload; fn new(config: StrategyConfig) -> Self { Self { processors: Vec::new(), config: config.clone() } @@ -103,12 +100,12 @@ impl Strategy for Liquidator { fn find_profitable_candidates< T: PoolTransaction, - DB: DBProvider + BlockHashReader + StateCommitmentProvider + DB: DBProvider + BlockHashReader + StateCommitmentProvider, >( &mut self, block: NumHash, latest_state_provider: LatestStateProviderRef<'_, DB>, - pending_txs: Vec + pending_txs: Vec, ) -> Result, AccessList)>, Error> { // TODO : define actions for find_profitable_candidates stage // 1. get liquidations candidates from db @@ -119,26 +116,27 @@ impl Strategy for Liquidator { } } - impl Liquidator { // add_processor adds indexer processor to liquidator fn add_processor(&mut self, table_name: String, processor_name: String) { let table = match TableName::from_str(&table_name) { Some(t) => t, - None => return, // Skip if table name is not recognized + None => { + return; + } // Skip if table name is not recognized }; let processor = match table { - TableName::DolomiteBorrowPositions => ProcessorInfo::new( - table.as_str(), - &processor_name, - |block_data, db| Box::pin(process_dolomite_borrow_positions(block_data, db)) - ), - TableName::AaveExecuteBorrow => ProcessorInfo::new( - table.as_str(), - &processor_name, - |block_data, db| Box::pin(process_aave_execute_borrow(block_data, db)) - ), + TableName::DolomiteBorrowPositions => { + ProcessorInfo::new(table.as_str(), &processor_name, |block_data, db| { + Box::pin(process_dolomite_borrow_positions(block_data, db)) + }) + } + TableName::AaveExecuteBorrow => { + ProcessorInfo::new(table.as_str(), &processor_name, |block_data, db| { + Box::pin(process_aave_execute_borrow(block_data, db)) + }) + } }; self.processors.push(processor); } @@ -151,8 +149,7 @@ impl Liquidator { &self, blocks_and_receipts: impl Iterator, &Vec)>, db: &RocksDB, - ) -> Result<()> - { + ) -> Result<()> { // Convert the iterator items into owned values directly let blocks_and_receipts: Vec<_> = blocks_and_receipts .map(|(block, receipts)| (block.clone(), receipts.clone())) @@ -172,10 +169,8 @@ impl Liquidator { async fn process_block_data( &self, block_data: &(RecoveredBlock, Vec), - db: &RocksDB - ) -> Result<()> - where - { + db: &RocksDB, + ) -> Result<()> { let block_number = block_data.0.number; // Create a vector to store all processing tasks @@ -187,14 +182,14 @@ impl Liquidator { let processor_name = processor.processor_name.clone(); let processor_fn = processor.processor; let block_data = block_data.clone(); - let db = db.clone(); // Clone db before spawn - + let db = db.clone(); // Clone db before spawn + // Spawn the task let task = tokio::spawn(async move { let event_start_time = Instant::now(); match processor_fn(&block_data, &db).await { Ok(()) => Ok((processor_name, event_start_time.elapsed())), - Err(e) => Err((processor_name, e.to_string())) + Err(e) => Err((processor_name, e.to_string())), } }); @@ -228,25 +223,21 @@ impl Liquidator { if !event_results.is_empty() { let events_summary: Vec = event_results .iter() - .map(|(name, time)| { - format!("{}({}, {:.2}s)", name, 1, time.as_secs_f64()) - }) + .map(|(name, time)| format!("{}({}, {:.2}s)", name, 1, time.as_secs_f64())) .collect(); tracing::info!( "exex{{id=\"exex-indexer\"}}: Block {} processed - Events: [{}], Total records: {}", block_number, events_summary.join(", "), - total_records, + total_records ); } // Create a consolidated error log if !failed_events.is_empty() { - let failure_summary: Vec = failed_events - .iter() - .map(|(name, error)| format!("{}: {}", name, error)) - .collect(); + let failure_summary: Vec = + failed_events.iter().map(|(name, error)| format!("{}: {}", name, error)).collect(); tracing::warn!( "exex{{id=\"exex-indexer\"}}: Block {} failures - {}", @@ -257,4 +248,4 @@ impl Liquidator { Ok(()) } -} \ No newline at end of file +} diff --git a/crates/strategy/src/liquidator/mod.rs b/crates/strategy/src/liquidator/mod.rs index 6813e6d..af775d1 100644 --- a/crates/strategy/src/liquidator/mod.rs +++ b/crates/strategy/src/liquidator/mod.rs @@ -1,9 +1,9 @@ -mod liquidator; -mod db_writer; mod datasets; +mod db_writer; +mod liquidator; mod types; -pub use liquidator::*; -pub use db_writer::*; pub use datasets::*; +pub use db_writer::*; +pub use liquidator::*; pub use types::*; diff --git a/crates/strategy/src/liquidator/types.rs b/crates/strategy/src/liquidator/types.rs index 137d44f..e08d478 100644 --- a/crates/strategy/src/liquidator/types.rs +++ b/crates/strategy/src/liquidator/types.rs @@ -2,9 +2,33 @@ use alloy_sol_types::sol; sol! { #[derive(Debug)] - struct LiquidatorTodoAction { + struct LiquidationPayload { + uint8 lendingType; // Protocol type: 1 = AAVE V3, 2+ = future protocols + uint256 liquidateAmount; // Amount of debt to repay (e.g., 1000 ETH) + address liquidateToken; // Debt token address to repay (e.g., WETH, DAI) + address collateralToken; // Collateral token to receive (e.g., WBTC, USDC) + address lendingProtocol; // Protocol contract address (e.g., AAVE Pool) + address user; // User address to liquidate (must have HF < 1) + } + + function getProfit( + uint256 initialAmt, + address stableToken, // USDC or other stable token address + LiquidationPayload memory payload + ) external view returns (uint256 finalAmount); + + #[derive(Debug)] + struct LiquidationPayloads { uint256 amount; - address token; - address to; + LiquidationPayload payload; + address swapRouterBefore; + address swapRouterAfter; + bytes swapDataBefore; + bytes swapDataAfter; } -} \ No newline at end of file + + function execute( + address stableToken, + LiquidationPayloads[] memory liquidations + ) external returns (uint256 totalProfit); +} From b6ea49f8ff8b1935639c24c73655cd9d7e27fe89 Mon Sep 17 00:00:00 2001 From: suha jin <89185836+djm07073@users.noreply.github.com> Date: Wed, 20 Aug 2025 12:58:57 +0900 Subject: [PATCH 12/12] fix: use macros to refactor and delete unused liquidator exex --- bin/searcher-reth/src/macros.rs | 21 +++++++ bin/searcher-reth/src/main.rs | 46 +++++++------- crates/extension/Cargo.toml | 5 +- crates/extension/src/exex.rs | 103 -------------------------------- 4 files changed, 42 insertions(+), 133 deletions(-) create mode 100644 bin/searcher-reth/src/macros.rs diff --git a/bin/searcher-reth/src/macros.rs b/bin/searcher-reth/src/macros.rs new file mode 100644 index 0000000..108f2a8 --- /dev/null +++ b/bin/searcher-reth/src/macros.rs @@ -0,0 +1,21 @@ +#[macro_export] +macro_rules! install_strategy { + ( + $builder:expr, + $config:expr, + $wallet:expr, + $signal_manager:expr, + $exex_id:expr, + $strategy_type:ty + ) => {{ + use searcher_reth_extension::strategy::core::strategy::Strategy; + + let cfg = $config.read().unwrap().get_strategy($exex_id).unwrap(); + let strategy = <$strategy_type>::new(cfg); + let searcher_exex = searcher_reth_extension::exex::SearcherExEx::new( + $wallet.clone(), + $signal_manager.subscribe(), + ); + $builder.install_exex($exex_id, move |ctx| searcher_exex.run(ctx, strategy)) + }}; +} diff --git a/bin/searcher-reth/src/main.rs b/bin/searcher-reth/src/main.rs index d332b72..de23c60 100644 --- a/bin/searcher-reth/src/main.rs +++ b/bin/searcher-reth/src/main.rs @@ -1,3 +1,5 @@ +mod macros; + use std::sync::{Arc, RwLock}; use clap::Parser; @@ -5,12 +7,8 @@ use reth::chainspec::EthereumChainSpecParser; use reth_node_ethereum::EthereumNode; use reth_tracing::tracing::error; use searcher_reth_extension::{ - exex::{LiquidatorExEx, SearcherExEx}, relayer_pool::WalletPool, - strategy::{ - core::strategy::Strategy, liquidator::Liquidator, path_finding::PathFinder, - profit_reporter::init_reporter, - }, + strategy::{liquidator::Liquidator, path_finding::PathFinder, profit_reporter::init_reporter}, util::signal_manager::SignalManager, }; use searcher_reth_manager::{ @@ -37,28 +35,24 @@ fn main() -> eyre::Result<()> { std::process::exit(0); }); - // Install Exex for various strategies - let searcher_exex = SearcherExEx::new(wallet.clone(), signal_manager.subscribe()); - let liquidator_exex = LiquidatorExEx::new(wallet.clone(), signal_manager.subscribe()); - + // Install strategies let mut node_builder = builder.node(EthereumNode::default()); - - // Install PathFinder strategy - node_builder = node_builder.install_exex(PATH_FINDER_EXEX_ID, { - let path_finder_cfg = - config.clone().read().unwrap().get_strategy(PATH_FINDER_EXEX_ID).unwrap(); - let path_finder = PathFinder::new(path_finder_cfg); - move |ctx| searcher_exex.run(ctx, path_finder) - }); - - // Install Indexer - node_builder = node_builder.install_exex(LIQUIDATOR_EXEX_ID, { - let liquidator_cfg = - config.clone().read().unwrap().get_strategy(LIQUIDATOR_EXEX_ID).unwrap(); - let liquidator = Liquidator::new(liquidator_cfg); - move |ctx| liquidator_exex.run(ctx, liquidator) - }); - + node_builder = install_strategy!( + node_builder, + config, + wallet, + signal_manager, + PATH_FINDER_EXEX_ID, + PathFinder + ); + node_builder = install_strategy!( + node_builder, + config, + wallet, + signal_manager, + LIQUIDATOR_EXEX_ID, + Liquidator + ); // TODO: Add other strategies here as needed let handle = node_builder.launch().await?; handle.wait_for_node_exit().await diff --git a/crates/extension/Cargo.toml b/crates/extension/Cargo.toml index 138e9fc..7742f87 100644 --- a/crates/extension/Cargo.toml +++ b/crates/extension/Cargo.toml @@ -44,7 +44,4 @@ searcher-reth-strategy.workspace = true searcher-reth-manager.workspace = true rayon.workspace = true -reth-transaction-pool.workspace = true - -# postgres -#tokio-postgres = { version = "0.7.13", features = ["with-chrono-0_4"] } \ No newline at end of file +reth-transaction-pool.workspace = true \ No newline at end of file diff --git a/crates/extension/src/exex.rs b/crates/extension/src/exex.rs index 3ecd8bf..95484a1 100644 --- a/crates/extension/src/exex.rs +++ b/crates/extension/src/exex.rs @@ -16,109 +16,6 @@ use searcher_reth_manager::SignalType; use searcher_reth_strategy::core::strategy::Strategy; use tokio::sync::broadcast; -pub struct LiquidatorExEx { - pub wallet: Arc, - pub signal_rx: broadcast::Receiver, -} - -impl LiquidatorExEx { - pub fn new(wallet: Arc, signal_rx: broadcast::Receiver) -> Self { - Self { wallet, signal_rx } - } - - pub async fn run( - self, - mut ctx: ExExContext, - mut strategy: S, - ) -> Result>> - where - S: Strategy, - Node: FullNodeComponents, - Node::Pool: TransactionPool, - <::Provider as DatabaseProviderFactory>::Provider: - BlockHashReader + StateCommitmentProvider, - Node::Provider: BlockReaderIdExt + ReceiptProvider + AccountReader + ChainSpecProvider, - { - let wallet = self.wallet.clone(); - let signal_rx = self.signal_rx.resubscribe(); - let vault = strategy.get_vault(); - Ok(async move { - let relayer_pool = Arc::new( - RelayerPool::new(ctx.components.clone(), wallet, signal_rx, strategy.gas_config()) - .await?, - ); - let relayer_tx = relayer_pool.start().await?; - tracing::info!( - target: "reth-exex", - event = "relayer_pool_started", - "Starting Relayer Pool" - ); - strategy.prepare(ctx.components.network().chain_id()); - while let Some(notification) = ctx.notifications.next().await { - if let Ok(ExExNotification::ChainCommitted { new: chain }) = notification { - let block = chain.tip(); - let num_hash = block.num_hash(); - let bytecode = strategy.get_code(); - - let database_provider: <::Provider as DatabaseProviderFactory>::Provider = ctx - .provider() - .database_provider_ro()?; - let latest_state_provider = LatestStateProviderRef::new(&database_provider); - - // 1. find candidates to liquidate - // pending txs is not used for liquidatior strategy. pass empty vectors. - let profitable_candidates = strategy.find_profitable_candidates( - num_hash, - latest_state_provider, - Vec::::new(), - )?; - - tracing::info!( - target: "reth-exex", - event = "filter_candidates", - success = profitable_candidates.is_none(), - num_hash = ?num_hash, - "Profitable candidates found" - ); - - if profitable_candidates.is_none() { - ctx.events.send(ExExEvent::FinishedHeight(num_hash))?; - continue; - } - - // 2. send candidates to relayer pool for braodcasting trnasactions - let (calldata, access_list) = profitable_candidates.unwrap(); - let relayer_channel = relayer_tx.clone(); - tokio::spawn(async move { - let message = RelayerMessage { to: vault, calldata, access_list }; - let r = relayer_channel.send(message).await; - - match r { - Ok(_) => tracing::info!( - target: "reth-exex", - event = "send_candidates_to_relayer_pool", - success = true, - num_hash = ?num_hash, - "Successfully sent candidates to relayer pool" - ), - Err(e) => tracing::error!( - target: "reth-exex", - event = "send_candidates_to_relayer_pool", - success = false, - error = ?e, - num_hash = ?num_hash, - "Failed to send candidates to relayer pool" - ), - } - }); - ctx.events.send(ExExEvent::FinishedHeight(num_hash))?; - } - } - Ok(()) - }) - } -} - pub struct SearcherExEx { pub wallet: Arc, pub signal_rx: broadcast::Receiver,