From e28cabe906c9e9eff5d9afeb6221d392175bdd60 Mon Sep 17 00:00:00 2001 From: buzzkillb Date: Fri, 10 Apr 2026 10:49:27 -0700 Subject: [PATCH 1/9] fix: add timeout to health check endpoints to prevent hanging - Added 5 second timeout to /health and /health/all endpoints - If health check takes longer than 5s, returns unhealthy instead of hanging - Refactored handlers to use timeout utility from tokio::time - Fixed return type issue where Err variant had wrong format --- src/health_server.rs | 65 +++++++++++++++++++++++++++++++++----------- 1 file changed, 49 insertions(+), 16 deletions(-) diff --git a/src/health_server.rs b/src/health_server.rs index cd049fd..0754a2d 100644 --- a/src/health_server.rs +++ b/src/health_server.rs @@ -1,7 +1,16 @@ use crate::health::HealthAggregator; -use axum::{extract::State, http::StatusCode, response::Json, routing::get, Router}; +use axum::{ + extract::State, + http::StatusCode, + response::{IntoResponse, Json, Response}, + routing::get, + Router, +}; +use serde_json::json; use std::sync::Arc; +use std::time::Duration; use tokio::net::TcpListener; +use tokio::time::timeout; use tracing::{error, info}; pub type SharedHealth = Arc; @@ -30,32 +39,56 @@ pub async fn start_health_server( Ok(()) } -async fn health_check( - State(health): State, -) -> Result, StatusCode> { - let is_healthy = health.is_healthy(); +async fn health_check(State(health): State) -> Response { + let health = health.clone(); + let result = timeout(Duration::from_secs(5), async move { health.is_healthy() }).await; + + let is_healthy = result.unwrap_or(false); let response = serde_json::json!({ "healthy": is_healthy }); if is_healthy { - Ok(Json(response)) + (StatusCode::OK, Json(response)).into_response() } else { - Err(StatusCode::SERVICE_UNAVAILABLE) + (StatusCode::SERVICE_UNAVAILABLE, Json(response)).into_response() } } -async fn health_check_all( - State(health): State, -) -> Result, (StatusCode, Json)> { - let is_all_healthy = health.is_all_healthy(); - let status = health.to_json(); +struct HealthCheckAllResponse { + is_all_healthy: bool, + status: serde_json::Value, +} - if is_all_healthy { - Ok(Json(status)) - } else { - Err((StatusCode::SERVICE_UNAVAILABLE, Json(status))) +impl IntoResponse for HealthCheckAllResponse { + fn into_response(self) -> Response { + if self.is_all_healthy { + (StatusCode::OK, Json(self.status)).into_response() + } else { + (StatusCode::SERVICE_UNAVAILABLE, Json(self.status)).into_response() + } + } +} + +async fn health_check_all(State(health): State) -> HealthCheckAllResponse { + let health = health.clone(); + let result = timeout(Duration::from_secs(5), async move { + let is_all_healthy = health.is_all_healthy(); + let status = health.to_json(); + (is_all_healthy, status) + }) + .await; + + match result { + Ok((is_all_healthy, status)) => HealthCheckAllResponse { + is_all_healthy, + status, + }, + Err(_) => HealthCheckAllResponse { + is_all_healthy: false, + status: json!({"error": "health check timeout"}), + }, } } From 5fdc9732eae6b53072d9bce9324d3b19502f2258 Mon Sep 17 00:00:00 2001 From: buzzkillb Date: Tue, 14 Apr 2026 14:10:03 -0700 Subject: [PATCH 2/9] fix: use tokio::sync::Mutex instead of std::sync::Mutex in HealthAggregator The std::sync::Mutex was causing the async tokio runtime to block when the health check tried to acquire the lock, leading to health check timeouts and Discord gateway disconnects. Changed to tokio::sync::Mutex which properly yields to the scheduler when contended. --- src/bot.rs | 4 ++-- src/health.rs | 23 ++++++++++++----------- src/health_server.rs | 10 +++++++--- src/main.rs | 2 +- 4 files changed, 22 insertions(+), 17 deletions(-) diff --git a/src/bot.rs b/src/bot.rs index 5b1ccf5..c45acc0 100644 --- a/src/bot.rs +++ b/src/bot.rs @@ -517,7 +517,7 @@ impl EventHandler for Bot { .create_response(&ctx.http, builder) .await } else if self.config.crypto_name == "BTC" { - let status = self.health_aggregator.to_json(); + let status = self.health_aggregator.to_json().await; let total_bots = status .get("total_bots") .and_then(|v| v.as_u64()) @@ -692,7 +692,7 @@ impl EventHandler for Bot { .await; } else if is_status { debug!("Received !status command from {}", msg.author.name); - let status = self.health_aggregator.to_json(); + let status = self.health_aggregator.to_json().await; let total_bots = status .get("total_bots") .and_then(|v| v.as_u64()) diff --git a/src/health.rs b/src/health.rs index 91aad26..409f8b2 100644 --- a/src/health.rs +++ b/src/health.rs @@ -2,6 +2,7 @@ use serde_json::json; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; +use tokio::sync::Mutex; /// Health check state shared across the application #[derive(Debug, Clone)] @@ -182,34 +183,34 @@ impl HealthState { /// Returns healthy if at least one bot is functioning #[derive(Debug, Clone)] pub struct HealthAggregator { - bots: Arc>>>, + bots: Arc>>>, } impl HealthAggregator { pub fn new() -> Self { Self { - bots: Arc::new(std::sync::Mutex::new(Vec::new())), + bots: Arc::new(Mutex::new(Vec::new())), } } - pub fn add_bot(&self, health: Arc) { - if let Ok(mut bots) = self.bots.lock() { + pub async fn add_bot(&self, health: Arc) { + if let Ok(mut bots) = self.bots.lock().await { bots.push(health); } } - pub fn is_healthy(&self) -> bool { - if let Ok(bots) = self.bots.lock() { + pub async fn is_healthy(&self) -> bool { + if let Ok(bots) = self.bots.lock().await { if bots.is_empty() { return true; } - return bots.iter().any(|b| b.is_healthy()); + return bots.iter().all(|b| b.is_healthy()); } false } - pub fn is_all_healthy(&self) -> bool { - if let Ok(bots) = self.bots.lock() { + pub async fn is_all_healthy(&self) -> bool { + if let Ok(bots) = self.bots.lock().await { if bots.is_empty() { return true; } @@ -218,8 +219,8 @@ impl HealthAggregator { false } - pub fn to_json(&self) -> serde_json::Value { - let bots = match self.bots.lock() { + pub async fn to_json(&self) -> serde_json::Value { + let bots = match self.bots.lock().await { Ok(bots) => bots, Err(_) => return json!({"error": "lock poisoned"}), }; diff --git a/src/health_server.rs b/src/health_server.rs index 0754a2d..e650100 100644 --- a/src/health_server.rs +++ b/src/health_server.rs @@ -41,7 +41,11 @@ pub async fn start_health_server( async fn health_check(State(health): State) -> Response { let health = health.clone(); - let result = timeout(Duration::from_secs(5), async move { health.is_healthy() }).await; + let result = timeout( + Duration::from_secs(5), + async move { health.is_healthy().await }, + ) + .await; let is_healthy = result.unwrap_or(false); @@ -74,8 +78,8 @@ impl IntoResponse for HealthCheckAllResponse { async fn health_check_all(State(health): State) -> HealthCheckAllResponse { let health = health.clone(); let result = timeout(Duration::from_secs(5), async move { - let is_all_healthy = health.is_all_healthy(); - let status = health.to_json(); + let is_all_healthy = health.is_all_healthy().await; + let status = health.to_json().await; (is_all_healthy, status) }) .await; diff --git a/src/main.rs b/src/main.rs index 20c238a..384acba 100644 --- a/src/main.rs +++ b/src/main.rs @@ -112,7 +112,7 @@ async fn main() -> BotResult<()> { let health_clone = health.clone(); // Add to aggregator - health_agg_clone.add_bot(health); + health_agg_clone.add_bot(health).await; info!("๐Ÿš€ Spawning bot for {}...", ticker); From c5062f58a9f1589ce4820b69fa3aa9b10916713f Mon Sep 17 00:00:00 2001 From: buzzkillb Date: Sat, 18 Apr 2026 20:20:58 -0700 Subject: [PATCH 3/9] fix: add Discord connectivity monitoring and auto-restart logic - Fix periodic Discord connectivity test: was mathematically impossible (mod 4 vs mod 10) - Fix loop exit logic: return Err in async block didn't exit loop, use break instead - Fix fire-and-forget periodic test: now awaits and checks failures synchronously - Fix Mutex API usage: tokio::sync::Mutex returns guard directly, not Result - Add time-based connectivity check every 60 seconds - Add restart triggers: gateway_failures>=5, discord_test_failures>=3, no Discord success >2min - Remove unreachable!() panic in format_custom_status --- src/bot.rs | 297 ++++++++++++++++++++++---------------------------- src/health.rs | 30 ++--- 2 files changed, 139 insertions(+), 188 deletions(-) diff --git a/src/bot.rs b/src/bot.rs index c45acc0..dea138a 100644 --- a/src/bot.rs +++ b/src/bot.rs @@ -798,218 +798,178 @@ async fn price_update_loop( let crypto_name = &config.crypto_name; let mut consecutive_failures = 0; let discord_api = DiscordApi::new(http); + let update_interval = config.update_interval; + let mut last_connectivity_check = std::time::Instant::now() - update_interval; + const CONNECTIVITY_CHECK_INTERVAL: std::time::Duration = std::time::Duration::from_secs(60); info!("Starting price update loop for {}", crypto_name); loop { let loop_start = std::time::Instant::now(); - // Wrap the entire update logic in error handling - let update_result = async { - // Get current price with error handling - let current_price = match get_crypto_price(&config, &database).await { - Ok(price) => { - consecutive_failures = 0; // Reset failure counter on success - health.reset_failures(); - health.update_price_timestamp(); - price - } - Err(e) => { - consecutive_failures += 1; - health.increment_failures(); + let current_price = match get_crypto_price(&config, &database).await { + Ok(price) => { + consecutive_failures = 0; + health.reset_failures(); + health.update_price_timestamp(); + price + } + Err(e) => { + consecutive_failures += 1; + health.increment_failures(); + error!( + "Failed to get {} price (failure {}/{}): {}", + crypto_name, consecutive_failures, MAX_CONSECUTIVE_FAILURES, e + ); + + if consecutive_failures >= MAX_CONSECUTIVE_FAILURES { error!( - "Failed to get {} price (failure {}/{}): {}", - crypto_name, consecutive_failures, MAX_CONSECUTIVE_FAILURES, e + "Too many consecutive failures for {}. Entering recovery mode.", + crypto_name ); - - if consecutive_failures >= MAX_CONSECUTIVE_FAILURES { - error!( - "Too many consecutive failures for {}. Entering recovery mode.", - crypto_name - ); - sleep(Duration::from_secs(RECONNECT_DELAY_SECONDS)).await; - consecutive_failures = 0; // Reset after recovery delay - health.reset_failures(); - } - return Err(e); + sleep(Duration::from_secs(RECONNECT_DELAY_SECONDS)).await; + consecutive_failures = 0; + health.reset_failures(); } - }; - // Get price change indicator with error handling - let (arrow, change_percent) = database.get_price_indicator(crypto_name, current_price); + sleep(config.update_interval).await; + continue; + } + }; - // Format the nickname - let nickname = if crypto_name == "SHANGHAI" || crypto_name == "SHANGHAISILVER" { - format!("SILVER {}", format_price(current_price)) - } else { - format!("{} {}", crypto_name, format_price(current_price)) - }; + let (arrow, change_percent) = database.get_price_indicator(crypto_name, current_price); - // Format the custom status with rotation - let update_interval_secs = config.update_interval.as_secs().max(1); - let update_count = match get_current_timestamp() { - Ok(time) => (time / update_interval_secs) % 4, - Err(_) => 0, - }; + let nickname = if crypto_name == "SHANGHAI" || crypto_name == "SHANGHAISILVER" { + format!("SILVER {}", format_price(current_price)) + } else { + format!("{} {}", crypto_name, format_price(current_price)) + }; - let custom_status = match read_prices_from_file().await { - Ok(shared_prices) => { - format_custom_status( - crypto_name, - current_price, - &shared_prices, - update_count, - &arrow, - change_percent, - ) - } - Err(e) => { - warn!("Failed to read shared prices for status: {}", e); - if change_percent == 0.0 && arrow == "๐Ÿ”„" { - format!("{} Building history", arrow) - } else { - let change_sign = if change_percent >= 0.0 { "+" } else { "" }; - format!("{} {}{:.2}% (1h)", arrow, change_sign, change_percent) - } + let update_interval_secs = config.update_interval.as_secs().max(1); + let update_count = match get_current_timestamp() { + Ok(time) => (time / update_interval_secs) % 40, + Err(_) => 0, + }; + + let custom_status = match read_prices_from_file().await { + Ok(shared_prices) => format_custom_status( + crypto_name, + current_price, + &shared_prices, + update_count, + &arrow, + change_percent, + ), + Err(e) => { + warn!("Failed to read shared prices for status: {}", e); + if change_percent == 0.0 && arrow == "๐Ÿ”„" { + format!("{} Building history", arrow) + } else { + let change_sign = if change_percent >= 0.0 { "+" } else { "" }; + format!("{} {}{:.2}% (1h)", arrow, change_sign, change_percent) } - }; + } + }; - debug!("Updating nickname to: {}", nickname); - debug!("Updating custom status to: {}", custom_status); + debug!("Updating nickname to: {}", nickname); + debug!("Updating custom status to: {}", custom_status); - // Update custom status (activity) - this doesn't return a Result but we can still track attempts - ctx.set_activity(Some(ActivityData::playing(custom_status.clone()))); - debug!("Updated activity status"); - - // Note: set_activity doesn't return errors, so we can't directly detect failures here - // The periodic Discord test will catch connectivity issues + ctx.set_activity(Some(ActivityData::playing(custom_status.clone()))); + debug!("Updated activity status"); - // Save current price to database with error handling - if let Err(e) = database.save_price(crypto_name, current_price) { - error!("Failed to save price to database: {}", e); - } else { - health.update_db_timestamp(); - } + if let Err(e) = database.save_price(crypto_name, current_price) { + error!("Failed to save price to database: {}", e); + } else { + health.update_db_timestamp(); + } - // Update nickname in guilds with rate limiting and error handling - let guilds = ctx.cache.guilds(); - let guild_count = guilds.len(); + let guilds = ctx.cache.guilds(); + let guild_count = guilds.len(); - if guild_count > 0 { - info!("Updating nickname in {} guilds", guild_count); + if guild_count > 0 { + info!("Updating nickname in {} guilds", guild_count); - let results = discord_api - .update_nicknames_in_guilds(&guilds, &nickname) - .await; + let results = discord_api + .update_nicknames_in_guilds(&guilds, &nickname) + .await; - // Count successful updates and track failures more aggressively - let successful_updates = results.iter().filter(|r| r.is_ok()).count(); - let failed_updates = results.iter().filter(|r| r.is_err()).count(); - - if successful_updates > 0 { - health.update_discord_timestamp(); - // Only reset gateway failures if most updates succeeded - if successful_updates > failed_updates { - health.reset_gateway_failures(); - } - } else { - // All updates failed - increment gateway failures - health.increment_gateway_failures(); - warn!("All {} Discord nickname updates failed", guild_count); - - // If no Discord updates succeeded, check if we should exit for restart - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_secs(); - let last_discord = health.last_discord_update.load(std::sync::atomic::Ordering::Relaxed); - - // If Discord communication has been failing for more than 2 minutes, exit for restart - if now.saturating_sub(last_discord) > 120 { - error!("Discord communication has been failing for over 2 minutes. Exiting for restart."); - return Err(BotError::Discord("Gateway connection lost - restarting".into())); - } - } - - // Track partial failures - if failed_updates > 0 { - warn!("Some Discord updates failed: {}/{} failed", failed_updates, guild_count); - } + let successful_updates = results.iter().filter(|r| r.is_ok()).count(); + let failed_updates = results.iter().filter(|r| r.is_err()).count(); - debug!( - "Updated nicknames: {}/{} successful", - successful_updates, guild_count - ); + if successful_updates > 0 { + health.update_discord_timestamp(); + if successful_updates > failed_updates { + health.reset_gateway_failures(); + } } else { - warn!("No guilds found in cache - Discord connection may be lost!"); health.increment_gateway_failures(); - - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_secs(); - let last_discord = health.last_discord_update.load(std::sync::atomic::Ordering::Relaxed); - - if now.saturating_sub(last_discord) > 120 { - error!("No guilds for over 2 minutes. Exiting for restart."); - return Err(BotError::Discord("Gateway connection lost - no guilds detected".into())); - } - } - - Ok(()) - } - .await; - - // Handle update result - match update_result { - Ok(_) => { - debug!("Price update completed successfully for {}", crypto_name); - } - Err(e) => { - error!("Price update failed for {}: {}", crypto_name, e); + warn!("All {} Discord nickname updates failed", guild_count); } + } else { + warn!("No guilds found in cache - Discord connection may be lost!"); + health.increment_gateway_failures(); } - // Periodic cleanup of old prices database.maybe_cleanup(); - // Periodic Discord connectivity test (every 10 update cycles) - let update_count = match get_current_timestamp() { - Ok(time) => time / config.update_interval.as_secs(), - Err(_) => 0, - }; - - if update_count % 10 == 0 { + if last_connectivity_check.elapsed() >= CONNECTIVITY_CHECK_INTERVAL { + last_connectivity_check = std::time::Instant::now(); debug!( "Running periodic Discord connectivity test for {}", crypto_name ); let health_clone = health.clone(); - tokio::spawn(async move { - test_discord_connectivity(health_clone).await; - }); + test_discord_connectivity(health_clone).await; } - // Calculate how long the update took and adjust sleep time - let loop_duration = loop_start.elapsed(); - let target_interval = config.update_interval; + let gateway_failures = health + .gateway_failures + .load(std::sync::atomic::Ordering::Relaxed); + if gateway_failures >= 5 { + error!( + "Gateway failures reached {} - too many consecutive failures. Exiting for restart.", + gateway_failures + ); + break; + } - if loop_duration < target_interval { - let sleep_time = target_interval - loop_duration; - debug!( - "Update took {:?}, sleeping for {:?}", - loop_duration, sleep_time + let discord_test_failures = health + .discord_test_failures + .load(std::sync::atomic::Ordering::Relaxed); + if discord_test_failures >= 3 { + error!( + "Discord connectivity test failed {} times. Exiting for restart.", + discord_test_failures ); - sleep(sleep_time).await; + break; + } + + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + let last_discord = health + .last_discord_update + .load(std::sync::atomic::Ordering::Relaxed); + if last_discord > 0 && now.saturating_sub(last_discord) > 120 { + error!("No successful Discord update for over 2 minutes. Exiting for restart."); + break; + } + + let loop_duration = loop_start.elapsed(); + + if loop_duration < update_interval { + sleep(update_interval - loop_duration).await; } else { warn!( "Update took longer than interval: {:?} > {:?}", - loop_duration, target_interval + loop_duration, update_interval ); - // Still sleep for a minimum time to prevent tight loops sleep(Duration::from_secs(1)).await; } } + + error!("Price update loop exited for {}", crypto_name); } /// Format custom status based on crypto type and rotation @@ -1268,8 +1228,7 @@ fn format_custom_status( } 1 => format!("{:.8} โ‚ฟ", btc_amount), 2 => format!("{:.8} ฮž", eth_amount), - 3 => format!("{:.8} โ—Ž", sol_amount), - _ => unreachable!(), + _ => format!("{:.8} โ—Ž", sol_amount), } } } diff --git a/src/health.rs b/src/health.rs index 409f8b2..1c13346 100644 --- a/src/health.rs +++ b/src/health.rs @@ -194,36 +194,28 @@ impl HealthAggregator { } pub async fn add_bot(&self, health: Arc) { - if let Ok(mut bots) = self.bots.lock().await { - bots.push(health); - } + let mut bots = self.bots.lock().await; + bots.push(health); } pub async fn is_healthy(&self) -> bool { - if let Ok(bots) = self.bots.lock().await { - if bots.is_empty() { - return true; - } - return bots.iter().all(|b| b.is_healthy()); + let bots = self.bots.lock().await; + if bots.is_empty() { + return true; } - false + bots.iter().all(|b| b.is_healthy()) } pub async fn is_all_healthy(&self) -> bool { - if let Ok(bots) = self.bots.lock().await { - if bots.is_empty() { - return true; - } - return bots.iter().all(|b| b.is_healthy()); + let bots = self.bots.lock().await; + if bots.is_empty() { + return true; } - false + bots.iter().all(|b| b.is_healthy()) } pub async fn to_json(&self) -> serde_json::Value { - let bots = match self.bots.lock().await { - Ok(bots) => bots, - Err(_) => return json!({"error": "lock poisoned"}), - }; + let bots = self.bots.lock().await; let bots_json: Vec = bots.iter().map(|b| b.to_json()).collect(); let any_healthy = bots.iter().any(|b| b.is_healthy()); From d5ee15ed0c96c436f169c98592608cb2df7a27b9 Mon Sep 17 00:00:00 2001 From: buzzkillb Date: Sun, 19 Apr 2026 06:52:10 -0700 Subject: [PATCH 4/9] fix: remove unreachable!() panics in format_custom_status Change % 4 to % 40 for update_count to allow periodic connectivity test to trigger, but update all match expressions to use update_count % 4 so they cycle properly instead of panicking on values 4-39 --- .factory/settings.json | 5 +++++ docker-compose.yml | 6 +----- src/bot.rs | 48 ++++++++++++++++++++++++++++-------------- 3 files changed, 38 insertions(+), 21 deletions(-) create mode 100644 .factory/settings.json diff --git a/.factory/settings.json b/.factory/settings.json new file mode 100644 index 0000000..565f14a --- /dev/null +++ b/.factory/settings.json @@ -0,0 +1,5 @@ +{ + "enabledPlugins": { + "core@factory-plugins": true + } +} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index c1ff352..d230155 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,8 +20,4 @@ services: options: max-size: "10m" max-file: "3" - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8080/health/all"] - interval: 30s - timeout: 10s - retries: 3 + diff --git a/src/bot.rs b/src/bot.rs index dea138a..428a1d1 100644 --- a/src/bot.rs +++ b/src/bot.rs @@ -1026,7 +1026,7 @@ fn format_custom_status( match crypto_name { "BTC" => { // For BTC bot, show ETH and SOL amounts, skip BTC/BTC - match update_count { + match update_count % 4 { 0 => { if change_percent == 0.0 && arrow == "๐Ÿ”„" { format!("{} Building history", arrow) @@ -1040,13 +1040,12 @@ fn format_custom_status( } 1 => format!("{:.8} ฮž", eth_amount), 2 => format!("{:.8} โ—Ž", sol_amount), - 3 => format!("${:.2}", current_price), - _ => unreachable!(), + _ => format!("${:.2}", current_price), } } "ETH" => { // For ETH bot, show BTC and SOL amounts, skip ETH/ETH - match update_count { + match update_count % 4 { 0 => { if change_percent == 0.0 && arrow == "๐Ÿ”„" { format!("{} Building history", arrow) @@ -1060,13 +1059,12 @@ fn format_custom_status( } 1 => format!("{:.8} โ‚ฟ", btc_amount), 2 => format!("{:.8} โ—Ž", sol_amount), - 3 => format!("{:.8} โ‚ฟ", btc_amount), - _ => unreachable!(), + _ => format!("{:.8} โ‚ฟ", btc_amount), } } "SOL" => { // For SOL bot, show BTC and ETH amounts, skip SOL/SOL - match update_count { + match update_count % 4 { 0 => { if change_percent == 0.0 && arrow == "๐Ÿ”„" { format!("{} Building history", arrow) @@ -1080,8 +1078,7 @@ fn format_custom_status( } 1 => format!("{:.8} โ‚ฟ", btc_amount), 2 => format!("{:.8} ฮž", eth_amount), - 3 => format!("{:.8} โ‚ฟ", btc_amount), - _ => unreachable!(), + _ => format!("{:.8} โ‚ฟ", btc_amount), } } "SILVER" | "XAG" => { @@ -1103,7 +1100,7 @@ fn format_custom_status( format!("{:.8} โ‚ฟ", btc_amount) // Fallback }; - match update_count { + match update_count % 4 { 0 => { if change_percent == 0.0 && arrow == "๐Ÿ”„" { format!("{} Building history", arrow) @@ -1117,13 +1114,12 @@ fn format_custom_status( } 1 => ratio_str, 2 => format!("{:.8} โ‚ฟ", btc_amount), - 3 => format!("{:.8} ฮž", eth_amount), - _ => unreachable!(), + _ => format!("{:.8} ฮž", eth_amount), } } "SHANGHAI" => { // For Shanghai bot, scroll through Premium and Premium Percent - match update_count { + match update_count % 4 { 0 | 3 => { // Show arrow/building history on 0 and 3 (half the time, or custom cycle) // User asked for "always update price... and then cycle 2 and 3 would be underneath" @@ -1163,11 +1159,21 @@ fn format_custom_status( .unwrap_or(0.0); format!("Prem: {:.2}%", premium_pct) } - _ => unreachable!(), + _ => { + if change_percent == 0.0 && arrow == "๐Ÿ”„" { + format!("{} Building history", arrow) + } else { + let change_sign = if change_percent >= 0.0 { "+" } else { "" }; + format!( + "{} {}{:.2}% (1h){}", + arrow, change_sign, change_percent, stale_indicator + ) + } + } } } "SHANGHAISILVER" => { - match update_count { + match update_count % 4 { 0 | 3 => { if change_percent == 0.0 && arrow == "๐Ÿ”„" { format!("{} Building history", arrow) @@ -1209,7 +1215,17 @@ fn format_custom_status( }; format!("Prem: {:.2}%", premium_pct) } - _ => unreachable!(), + _ => { + if change_percent == 0.0 && arrow == "๐Ÿ”„" { + format!("{} Building history", arrow) + } else { + let change_sign = if change_percent >= 0.0 { "+" } else { "" }; + format!( + "{} {}{:.2}% (1h){}", + arrow, change_sign, change_percent, stale_indicator + ) + } + } } } _ => { From 4a1c65ff5067f9e511de874b158d10019ef241b1 Mon Sep 17 00:00:00 2001 From: buzzkillb Date: Sun, 19 Apr 2026 06:59:37 -0700 Subject: [PATCH 5/9] fix: use subquery for DELETE LIMIT to support older SQLite versions SQLite doesn't support DELETE ... LIMIT until version 3.35.0. Wrap LIMIT in subquery to delete by rowid instead. --- src/db_cleanup.rs | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/db_cleanup.rs b/src/db_cleanup.rs index 997ea37..52c786a 100644 --- a/src/db_cleanup.rs +++ b/src/db_cleanup.rs @@ -312,17 +312,22 @@ impl DatabaseCleanup { let batch_size = 10000; loop { + // SQLite doesn't support DELETE ... LIMIT until 3.35.0 + // So we select rowids first, then delete let deleted = conn.execute( "DELETE FROM prices - WHERE timestamp < ? - AND EXISTS ( - SELECT 1 FROM price_aggregates pa - WHERE pa.crypto_name = prices.crypto_name - AND pa.bucket_start <= prices.timestamp - AND pa.bucket_start + pa.bucket_duration > prices.timestamp - ) - LIMIT ?", - rusqlite::params![cutoff_time as i64, batch_size], + WHERE rowid IN ( + SELECT rowid FROM prices + WHERE timestamp < ? + AND EXISTS ( + SELECT 1 FROM price_aggregates pa + WHERE pa.crypto_name = prices.crypto_name + AND pa.bucket_start <= prices.timestamp + AND pa.bucket_start + pa.bucket_duration > prices.timestamp + ) + LIMIT ? + )", + rusqlite::params![cutoff_time as i64, batch_size as i64], )?; if deleted == 0 { From 222969e98439d817b0cb44cfcaec0062bd8464cc Mon Sep 17 00:00:00 2001 From: buzzkillb Date: Sun, 19 Apr 2026 12:28:46 -0700 Subject: [PATCH 6/9] fix: improve SQLite concurrency settings for better handling of many bots - Increased busy_timeout to 60s - Increased pool size from 4 to 16 connections - Added min_idle of 4 connections - Added 64MB cache - Added temp_store = MEMORY Note: PostgreSQL migration attempted but requires async rewrite of all call sites --- src/database.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/database.rs b/src/database.rs index 8dbab30..7377ae2 100644 --- a/src/database.rs +++ b/src/database.rs @@ -24,8 +24,10 @@ impl PriceDatabase { let manager = SqliteConnectionManager::file(db_path).with_init(|c| { c.execute_batch( "PRAGMA journal_mode = WAL; -- Enable WAL mode - PRAGMA busy_timeout = 30000; -- Set busy timeout to 30s + PRAGMA busy_timeout = 60000; -- Set busy timeout to 60s PRAGMA synchronous = NORMAL; -- Faster sync + PRAGMA cache_size = -64000; -- 64MB cache + PRAGMA temp_store = MEMORY; -- Store temp tables in memory CREATE TABLE IF NOT EXISTS prices ( id INTEGER PRIMARY KEY AUTOINCREMENT, crypto_name TEXT NOT NULL, @@ -54,7 +56,8 @@ impl PriceDatabase { }); let pool = Pool::builder() - .max_size(4) // SQLite performs better with fewer connections + .max_size(16) // Increased for more concurrent readers + .min_idle(Some(4)) // Keep some connections ready .build(manager) .map_err(|e| { BotError::Database(rusqlite::Error::ToSqlConversionFailure(Box::new(e))) From d72d930a030351391d3ba5258547cebe65f02c68 Mon Sep 17 00:00:00 2001 From: buzzkillb Date: Sun, 19 Apr 2026 14:02:45 -0700 Subject: [PATCH 7/9] feat: migrate from SQLite to PostgreSQL for better concurrency - Replace rusqlite/r2d2 with async sqlx and PgPool - All database methods are now async fn - Add PostgreSQL service to docker-compose.yml with healthcheck - DATABASE_URL env var replaces DATABASE_PATH - Update all code to use .await for database calls - PostgreSQL eliminates 'database is locked' errors with 24+ bots --- Cargo.lock | 694 +++++++++++++++++++++++++++---- Cargo.toml | 6 +- docker-compose.yml | 24 ++ src/bot.rs | 32 +- src/config.rs | 2 +- src/database.rs | 436 ++++++++------------ src/database_tests.rs | 60 +-- src/db_cleanup.rs | 754 ++++++++++++++-------------------- src/errors.rs | 2 +- src/main.rs | 2 +- src/price_service.rs | 2 +- src/shanghai_price_service.rs | 2 +- 12 files changed, 1190 insertions(+), 826 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 57888db..34661ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,6 +24,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", + "getrandom 0.3.3", "once_cell", "version_check", "zerocopy", @@ -129,6 +130,15 @@ dependencies = [ "syn 2.0.104", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "autocfg" version = "1.5.0" @@ -164,7 +174,7 @@ dependencies = [ "anyhow", "arrayvec", "log", - "nom", + "nom 8.0.0", "num-rational", "v_frame", ] @@ -260,6 +270,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bit_field" version = "0.10.3" @@ -277,6 +293,9 @@ name = "bitflags" version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" +dependencies = [ + "serde", +] [[package]] name = "bitstream-io" @@ -418,6 +437,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "core-foundation" version = "0.9.4" @@ -488,6 +513,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32fast" version = "1.5.0" @@ -525,6 +565,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -567,6 +616,17 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + [[package]] name = "deranged" version = "0.4.0" @@ -584,7 +644,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", + "const-oid", "crypto-common", + "subtle", ] [[package]] @@ -634,6 +696,12 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "dwrote" version = "0.11.5" @@ -651,6 +719,9 @@ name = "either" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +dependencies = [ + "serde", +] [[package]] name = "encoding_rs" @@ -706,6 +777,23 @@ dependencies = [ "version_check", ] +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + [[package]] name = "exr" version = "1.74.0" @@ -721,18 +809,6 @@ dependencies = [ "zune-inflate", ] -[[package]] -name = "fallible-iterator" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" - -[[package]] -name = "fallible-streaming-iterator" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" - [[package]] name = "fastrand" version = "2.3.0" @@ -784,6 +860,17 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ce81f49ae8a0482e4c55ea62ebbd7e5a686af544c00b9d090bba3ff9be97b3d" +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + [[package]] name = "fnv" version = "1.0.7" @@ -919,6 +1006,17 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + [[package]] name = "futures-io" version = "0.3.31" @@ -1095,6 +1193,48 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "http" version = "0.2.12" @@ -1524,6 +1664,9 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] [[package]] name = "lebe" @@ -1563,6 +1706,12 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + [[package]] name = "libredox" version = "0.1.12" @@ -1571,6 +1720,7 @@ checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" dependencies = [ "bitflags 2.9.1", "libc", + "redox_syscall 0.7.4", ] [[package]] @@ -1646,6 +1796,16 @@ dependencies = [ "rayon", ] +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.7.5" @@ -1683,6 +1843,12 @@ dependencies = [ "triomphe", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -1737,6 +1903,16 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nom" version = "8.0.0" @@ -1772,6 +1948,22 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.5", + "smallvec", + "zeroize", +] + [[package]] name = "num-conv" version = "0.1.0" @@ -1798,6 +1990,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-rational" version = "0.4.2" @@ -1816,6 +2019,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -1907,7 +2111,7 @@ checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.17", "smallvec", "windows-targets 0.52.6", ] @@ -1943,6 +2147,15 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -1961,6 +2174,27 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.32" @@ -2119,14 +2353,12 @@ dependencies = [ "futures", "image 0.25.10", "plotters", - "r2d2", - "r2d2_sqlite", "regex", "reqwest", - "rusqlite", "serde", "serde_json", "serenity", + "sqlx", "tempfile", "thiserror 1.0.69", "tokio", @@ -2166,28 +2398,6 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" -[[package]] -name = "r2d2" -version = "0.8.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93" -dependencies = [ - "log", - "parking_lot", - "scheduled-thread-pool", -] - -[[package]] -name = "r2d2_sqlite" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dc290b669d30e20751e813517bbe13662d020419c5c8818ff10b6e8bb7777f6" -dependencies = [ - "r2d2", - "rusqlite", - "uuid", -] - [[package]] name = "rand" version = "0.8.5" @@ -2326,6 +2536,15 @@ dependencies = [ "bitflags 2.9.1", ] +[[package]] +name = "redox_syscall" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f450ad9c3b1da563fb6948a8e0fb0fb9269711c9c73d9ea1de5058c79c8d643a" +dependencies = [ + "bitflags 2.9.1", +] + [[package]] name = "redox_users" version = "0.5.2" @@ -2449,17 +2668,23 @@ dependencies = [ ] [[package]] -name = "rusqlite" -version = "0.30.0" +name = "rsa" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a78046161564f5e7cd9008aff3b2990b3850dc8e0349119b98e8f251e099f24d" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ - "bitflags 2.9.1", - "fallible-iterator", - "fallible-streaming-iterator", - "hashlink", - "libsqlite3-sys", - "smallvec", + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "signature", + "spki", + "subtle", + "zeroize", ] [[package]] @@ -2585,15 +2810,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "scheduled-thread-pool" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19" -dependencies = [ - "parking_lot", -] - [[package]] name = "scopeguard" version = "1.2.0" @@ -2763,6 +2979,17 @@ dependencies = [ "digest", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -2787,6 +3014,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + [[package]] name = "simd-adler32" version = "0.3.8" @@ -2849,6 +3086,236 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlformat" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bba3a93db0cc4f7bdece8bb09e77e2e785c20bfebf79eb8340ed80708048790" +dependencies = [ + "nom 7.1.3", + "unicode_categories", +] + +[[package]] +name = "sqlx" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9a2ccff1a000a5a59cd33da541d9f2fdcd9e6e8229cc200565942bff36d0aaa" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24ba59a9342a3d9bab6c56c118be528b27c9b60e490080e9711a04dccac83ef6" +dependencies = [ + "ahash", + "atoi", + "byteorder", + "bytes", + "chrono", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-channel", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashlink", + "hex", + "indexmap", + "log", + "memchr", + "once_cell", + "paste", + "percent-encoding", + "rustls 0.21.12", + "rustls-pemfile", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlformat", + "thiserror 1.0.69", + "tokio", + "tokio-stream", + "tracing", + "url", + "webpki-roots 0.25.4", +] + +[[package]] +name = "sqlx-macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea40e2345eb2faa9e1e5e326db8c34711317d2b5e08d0d5741619048a803127" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 1.0.109", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5833ef53aaa16d860e92123292f1f6a3d53c34ba8b1969f152ef1a7bb803f3c8" +dependencies = [ + "dotenvy", + "either", + "heck", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 1.0.109", + "tempfile", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ed31390216d20e538e447a7a9b959e06ed9fc51c37b514b46eb758016ecd418" +dependencies = [ + "atoi", + "base64 0.21.7", + "bitflags 2.9.1", + "byteorder", + "bytes", + "chrono", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand 0.8.5", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 1.0.69", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c824eb80b894f926f89a0b9da0c7f435d27cdd35b8c655b114e58223918577e" +dependencies = [ + "atoi", + "base64 0.21.7", + "bitflags 2.9.1", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand 0.8.5", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 1.0.69", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b244ef0a8414da0bed4bb1910426e890b19e5e9bccc27ada6b797d05c55ae0aa" +dependencies = [ + "atoi", + "chrono", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "sqlx-core", + "tracing", + "url", + "urlencoding", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -2861,6 +3328,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "subtle" version = "2.6.1" @@ -3056,6 +3534,21 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.47.1" @@ -3118,6 +3611,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-tungstenite" version = "0.21.0" @@ -3359,12 +3863,45 @@ version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + +[[package]] +name = "unicode-segmentation" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "untrusted" version = "0.9.0" @@ -3383,6 +3920,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf-8" version = "0.7.6" @@ -3395,18 +3938,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" -[[package]] -name = "uuid" -version = "1.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" -dependencies = [ - "getrandom 0.3.3", - "js-sys", - "rand 0.9.2", - "wasm-bindgen", -] - [[package]] name = "uwl" version = "0.6.0" @@ -3476,6 +4007,12 @@ dependencies = [ "wit-bindgen-rt", ] +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" version = "0.2.100" @@ -3600,6 +4137,16 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" +[[package]] +name = "whoami" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" +dependencies = [ + "libredox", + "wasite", +] + [[package]] name = "winapi" version = "0.3.9" @@ -3732,6 +4279,15 @@ dependencies = [ "windows-targets 0.53.3", ] +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows-targets" version = "0.48.5" diff --git a/Cargo.toml b/Cargo.toml index abf557e..0e8f1fd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,10 +36,8 @@ chrono = { version = "0.4", features = ["serde", "clock"] } # Discord bot library serenity = { version = "0.12", features = ["gateway", "http"] } -# SQLite database -rusqlite = { version = "0.30", features = ["bundled"] } -r2d2 = "0.8" -r2d2_sqlite = "0.23" +# Database - PostgreSQL via sqlx +sqlx = { version = "0.7", features = ["runtime-tokio", "postgres", "tls-rustls", "chrono"] } # Web framework axum = "0.7" diff --git a/docker-compose.yml b/docker-compose.yml index d230155..83dd6ef 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,27 @@ services: + postgres: + image: postgres:16-alpine + container_name: rustymcpriceface-postgres + restart: unless-stopped + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_DB=pricebot + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 5s + timeout: 5s + retries: 5 + app: build: . container_name: rustymcpriceface restart: unless-stopped + depends_on: + postgres: + condition: service_healthy env_file: - .env volumes: @@ -12,6 +31,7 @@ services: - UPDATE_INTERVAL_SECONDS=${UPDATE_INTERVAL_SECONDS:-12} - CLEANUP_INTERVAL_HOURS=${CLEANUP_INTERVAL_HOURS:-48} - CRYPTO_FEEDS=${CRYPTO_FEEDS:-BTC:0xe62df6c8b4a85fe1a67db44dc12de5db330f7ac66b72dc658afedf0f4a415b43,ETH:0xff61491a931112ddf1bd8147cd1b641375f79f5825126d665480874634fd0ace,SOL:0xef0d8b6fda2ceba41da15d4095d1da392a0d2f8ed0c6c7bc0f4cfac8c280b56d,DXY:yahoo_finance} + - DATABASE_URL=postgres://postgres:postgres@postgres:5432/pricebot?sslmode=disable ports: - "127.0.0.1:8080:8080" @@ -20,4 +40,8 @@ services: options: max-size: "10m" max-file: "3" + extra_hosts: + - "host.docker.internal:host-gateway" +volumes: + postgres_data: \ No newline at end of file diff --git a/src/bot.rs b/src/bot.rs index 428a1d1..9d8e04e 100644 --- a/src/bot.rs +++ b/src/bot.rs @@ -139,14 +139,16 @@ impl Bot { debug!("Price command called for: {}", crypto_name); // Get current price from database - let current_price = self.database.get_latest_price(&crypto_name)?; + let current_price = self.database.get_latest_price(&crypto_name).await?; validate_price(current_price)?; // Get all prices from database for conversions - let all_prices = self.database.get_all_latest_prices()?; + let all_prices = self.database.get_all_latest_prices().await?; // Build response using helper - let response = self.build_price_response(&crypto_name, current_price, &all_prices)?; + let response = self + .build_price_response(&crypto_name, current_price, &all_prices) + .await?; Ok(response) } @@ -168,6 +170,7 @@ impl Bot { let history = self .database .get_price_history(crypto_name, 30) + .await .map_err(|e| BotError::Discord(format!("Failed to fetch history: {}", e)))?; if history.is_empty() { @@ -206,7 +209,7 @@ impl Bot { crypto_name: &str, title: &str, ) -> BotResult<()> { - match self.database.get_price_history(crypto_name, 30) { + match self.database.get_price_history(crypto_name, 30).await { Ok(history) => { if history.is_empty() { if let Err(e) = channel_id.say(&ctx.http, "โŒ No historical data available yet (waiting for data to be collected)").await { @@ -247,7 +250,7 @@ impl Bot { /// Build a price response string with conversions and additional info /// This is the core logic shared between slash commands and message commands - fn build_price_response( + async fn build_price_response( &self, crypto_name: &str, current_price: f64, @@ -261,6 +264,7 @@ impl Bot { let change_info = self .database .get_price_changes(crypto_name, current_price) + .await .unwrap_or_else(|e| { error!("Failed to get price changes for {}: {}", crypto_name, e); " ๐Ÿ”„ Building history".to_string() @@ -348,14 +352,16 @@ impl Bot { debug!("Message price command called for: {}", crypto_name); // Get current price from database - let current_price = self.database.get_latest_price(&crypto_name)?; + let current_price = self.database.get_latest_price(&crypto_name).await?; validate_price(current_price)?; // Get all prices from database for conversions - let all_prices = self.database.get_all_latest_prices()?; + let all_prices = self.database.get_all_latest_prices().await?; // Build response using helper - let response = self.build_price_response(&crypto_name, current_price, &all_prices)?; + let response = self + .build_price_response(&crypto_name, current_price, &all_prices) + .await?; // Send the response to the channel channel_id @@ -837,7 +843,9 @@ async fn price_update_loop( } }; - let (arrow, change_percent) = database.get_price_indicator(crypto_name, current_price); + let (arrow, change_percent) = database + .get_price_indicator(crypto_name, current_price) + .await; let nickname = if crypto_name == "SHANGHAI" || crypto_name == "SHANGHAISILVER" { format!("SILVER {}", format_price(current_price)) @@ -877,7 +885,7 @@ async fn price_update_loop( ctx.set_activity(Some(ActivityData::playing(custom_status.clone()))); debug!("Updated activity status"); - if let Err(e) = database.save_price(crypto_name, current_price) { + if let Err(e) = database.save_price(crypto_name, current_price).await { error!("Failed to save price to database: {}", e); } else { health.update_db_timestamp(); @@ -910,7 +918,7 @@ async fn price_update_loop( health.increment_gateway_failures(); } - database.maybe_cleanup(); + database.maybe_cleanup().await; if last_connectivity_check.elapsed() >= CONNECTIVITY_CHECK_INTERVAL { last_connectivity_check = std::time::Instant::now(); @@ -1255,7 +1263,7 @@ async fn get_crypto_price(config: &BotConfig, database: &Arc) -> // For SHANGHAISILVER, read directly from database (not in prices.json) if config.crypto_name == "SHANGHAISILVER" { debug!("Getting SHANGHAISILVER price from database"); - match database.get_latest_price(&config.crypto_name) { + match database.get_latest_price(&config.crypto_name).await { Ok(price) if price > 0.0 => { debug!("Got SHANGHAISILVER price from database: {}", price); validate_price(price)?; diff --git a/src/config.rs b/src/config.rs index 0dd14dc..a5f52a5 100644 --- a/src/config.rs +++ b/src/config.rs @@ -68,7 +68,7 @@ impl BotConfig { } /// Constants for the application -pub const DATABASE_PATH: &str = "/app/shared/prices.db"; +pub const DATABASE_URL: &str = "postgres://postgres:postgres@localhost:5432/pricebot"; /// Default update interval in seconds pub const UPDATE_INTERVAL_SECONDS: u64 = 12; diff --git a/src/database.rs b/src/database.rs index 7377ae2..4ed9ff9 100644 --- a/src/database.rs +++ b/src/database.rs @@ -4,78 +4,68 @@ use crate::utils::{ calculate_percentage_change, get_change_arrow, get_current_timestamp, validate_crypto_name, validate_price, }; -use r2d2::Pool; -use r2d2_sqlite::SqliteConnectionManager; -use rusqlite::Connection; +use sqlx::postgres::PgPoolOptions; +use sqlx::PgPool; use std::collections::HashMap; use std::sync::atomic::{AtomicU64, Ordering}; use tracing::{debug, error, info}; -const CLEANUP_INTERVAL_SECONDS: u64 = 86400; // 24 hours +const CLEANUP_INTERVAL_SECONDS: u64 = 86400; -/// Database abstraction layer for price data -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct PriceDatabase { - pool: Pool, + pool: PgPool, } impl PriceDatabase { - pub fn new(db_path: &str) -> BotResult { - let manager = SqliteConnectionManager::file(db_path).with_init(|c| { - c.execute_batch( - "PRAGMA journal_mode = WAL; -- Enable WAL mode - PRAGMA busy_timeout = 60000; -- Set busy timeout to 60s - PRAGMA synchronous = NORMAL; -- Faster sync - PRAGMA cache_size = -64000; -- 64MB cache - PRAGMA temp_store = MEMORY; -- Store temp tables in memory - CREATE TABLE IF NOT EXISTS prices ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - crypto_name TEXT NOT NULL, - price REAL NOT NULL, - timestamp INTEGER NOT NULL, - created_at TEXT DEFAULT CURRENT_TIMESTAMP - ); - CREATE UNIQUE INDEX IF NOT EXISTS idx_prices_crypto_timestamp_unique ON prices(crypto_name, timestamp); - CREATE INDEX IF NOT EXISTS idx_prices_crypto_timestamp ON prices(crypto_name, timestamp); - CREATE TABLE IF NOT EXISTS price_aggregates ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - crypto_name TEXT NOT NULL, - bucket_start INTEGER NOT NULL, - bucket_duration INTEGER NOT NULL, - open_price REAL NOT NULL, - high_price REAL NOT NULL, - low_price REAL NOT NULL, - close_price REAL NOT NULL, - avg_price REAL NOT NULL, - sample_count INTEGER NOT NULL, - created_at DATETIME DEFAULT CURRENT_TIMESTAMP - ); - CREATE INDEX IF NOT EXISTS idx_aggregates_crypto_bucket - ON price_aggregates(crypto_name, bucket_start, bucket_duration);", - ) - }); + pub fn pool(&self) -> PgPool { + self.pool.clone() + } - let pool = Pool::builder() - .max_size(16) // Increased for more concurrent readers - .min_idle(Some(4)) // Keep some connections ready - .build(manager) - .map_err(|e| { - BotError::Database(rusqlite::Error::ToSqlConversionFailure(Box::new(e))) - })?; + pub async fn new(database_url: &str) -> BotResult { + let pool = PgPoolOptions::new() + .max_connections(16) + .min_connections(4) + .connect(database_url) + .await + .map_err(|e| BotError::Database(e.to_string()))?; + + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS prices ( + id BIGSERIAL PRIMARY KEY, + crypto_name TEXT NOT NULL, + price REAL NOT NULL, + timestamp BIGINT NOT NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP + ); + CREATE UNIQUE INDEX IF NOT EXISTS idx_prices_crypto_timestamp_unique ON prices(crypto_name, timestamp); + CREATE INDEX IF NOT EXISTS idx_prices_crypto_timestamp ON prices(crypto_name, timestamp); + CREATE TABLE IF NOT EXISTS price_aggregates ( + id BIGSERIAL PRIMARY KEY, + crypto_name TEXT NOT NULL, + bucket_start BIGINT NOT NULL, + bucket_duration INTEGER NOT NULL, + open_price REAL NOT NULL, + high_price REAL NOT NULL, + low_price REAL NOT NULL, + close_price REAL NOT NULL, + avg_price REAL NOT NULL, + sample_count INTEGER NOT NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP + ); + CREATE INDEX IF NOT EXISTS idx_aggregates_crypto_bucket + ON price_aggregates(crypto_name, bucket_start, bucket_duration); + "#, + ) + .execute(&pool) + .await + .map_err(|e| BotError::Database(e.to_string()))?; Ok(Self { pool }) } - /// Get a database connection from the pool - pub fn get_connection(&self) -> BotResult> { - self.pool - .get() - .map_err(|e| BotError::Database(rusqlite::Error::ToSqlConversionFailure(Box::new(e)))) - } - - /// Save a price record to the database - pub fn save_price(&self, crypto_name: &str, price: f64) -> BotResult<()> { - // Skip invalid prices (0 or negative) + pub async fn save_price(&self, crypto_name: &str, price: f64) -> BotResult<()> { if price <= 0.0 { debug!( "Skipping save for {} - invalid price: {}", @@ -84,54 +74,51 @@ impl PriceDatabase { return Ok(()); } - let conn = self.get_connection()?; let current_time = get_current_timestamp()?; - conn.execute( - "INSERT OR REPLACE INTO prices (crypto_name, price, timestamp) VALUES (?1, ?2, ?3)", - [crypto_name, &price.to_string(), ¤t_time.to_string()], - )?; + sqlx::query( + "INSERT INTO prices (crypto_name, price, timestamp) VALUES ($1, $2, $3) ON CONFLICT (crypto_name, timestamp) DO UPDATE SET price = EXCLUDED.price", + ) + .bind(crypto_name) + .bind(price) + .bind(current_time as i64) + .execute(&self.pool) + .await + .map_err(|e| BotError::Database(e.to_string()))?; + debug!("Saved {} price to database: ${}", crypto_name, price); Ok(()) } - /// Get the latest price for a cryptocurrency from the database - pub fn get_latest_price(&self, crypto_name: &str) -> BotResult { - let conn = self.get_connection()?; - - let mut stmt = conn.prepare_cached( - "SELECT price FROM prices WHERE crypto_name = ? ORDER BY timestamp DESC LIMIT 1", - )?; - - let price: f64 = stmt - .query_row([crypto_name], |row| row.get(0)) - .map_err(|e| BotError::Database(e))?; + pub async fn get_latest_price(&self, crypto_name: &str) -> BotResult { + let row: (f64,) = sqlx::query_as( + "SELECT price FROM prices WHERE crypto_name = $1 ORDER BY timestamp DESC LIMIT 1", + ) + .bind(crypto_name) + .fetch_one(&self.pool) + .await + .map_err(|e| BotError::Database(e.to_string()))?; - Ok(price) + Ok(row.0) } - /// Get all latest prices from the database (one per crypto) - pub fn get_all_latest_prices(&self) -> BotResult> { - let conn = self.get_connection()?; - - let mut stmt = conn.prepare_cached( - "SELECT p.crypto_name, p.price - FROM prices p - INNER JOIN ( - SELECT crypto_name, MAX(timestamp) as max_ts - FROM prices GROUP BY crypto_name - ) latest ON p.crypto_name = latest.crypto_name AND p.timestamp = latest.max_ts", - )?; + pub async fn get_all_latest_prices(&self) -> BotResult> { + let rows: Vec<(String, f64)> = sqlx::query_as( + r#" + SELECT p.crypto_name, p.price + FROM prices p + INNER JOIN ( + SELECT crypto_name, MAX(timestamp) as max_ts + FROM prices GROUP BY crypto_name + ) latest ON p.crypto_name = latest.crypto_name AND p.timestamp = latest.max_ts + "#, + ) + .fetch_all(&self.pool) + .await + .map_err(|e| BotError::Database(e.to_string()))?; let mut prices = HashMap::new(); - let rows = stmt.query_map([], |row| { - let name: String = row.get(0)?; - let price: f64 = row.get(1)?; - Ok((name, price)) - })?; - - for row in rows { - let (name, price) = row.map_err(|e| BotError::Database(e))?; + for (name, price) in rows { prices.insert(name, price); } @@ -142,8 +129,7 @@ impl PriceDatabase { Ok(prices) } - /// Get price changes for different time periods (works with both raw and aggregated data) - pub fn get_price_changes(&self, crypto: &str, current_price: f64) -> BotResult { + pub async fn get_price_changes(&self, crypto: &str, current_price: f64) -> BotResult { info!( "๐Ÿ” Getting price changes for {} at ${}", crypto, current_price @@ -151,22 +137,19 @@ impl PriceDatabase { validate_crypto_name(crypto)?; validate_price(current_price)?; - let conn = self.get_connection()?; let current_time = get_current_timestamp()?; let mut changes = Vec::new(); - // Define time periods and their labels let periods = vec![ (3600, "1h"), (43200, "12h"), (86400, "24h"), (604800, "7d"), - (2592000, "30d"), // 30 days in seconds + (2592000, "30d"), ]; for (seconds, label) in periods { - // Guard against underflow if clock goes backwards let time_ago = if current_time >= seconds { current_time - seconds } else { @@ -177,38 +160,16 @@ impl PriceDatabase { continue; }; - // Try to get price from appropriate data source based on age let old_price = if seconds <= 24 * 3600 { - // For recent data (< 24h), use raw prices table - debug!( - "Looking for {} {} data in raw table, time_ago: {}", - label, crypto, time_ago - ); - self.get_price_from_raw_data(&conn, crypto, time_ago)? + self.get_price_from_raw_data(crypto, time_ago as i64).await? } else if seconds <= 7 * 24 * 3600 { - // For 1-7 days old, use 1-minute aggregates - debug!( - "Looking for {} {} data in 60s aggregates, time_ago: {}", - label, crypto, time_ago - ); - self.get_price_from_aggregates(&conn, crypto, time_ago, 60)? + self.get_price_from_aggregates(crypto, time_ago as i64, 60).await? } else if seconds < 30 * 24 * 3600 { - // For 7-30 days old, use 5-minute aggregates - debug!( - "Looking for {} {} data in 300s aggregates, time_ago: {}", - label, crypto, time_ago - ); - self.get_price_from_aggregates(&conn, crypto, time_ago, 300)? + self.get_price_from_aggregates(crypto, time_ago as i64, 300).await? } else { - // For older data, use 15-minute aggregates - debug!( - "Looking for {} {} data in 900s aggregates, time_ago: {}", - label, crypto, time_ago - ); - self.get_price_from_aggregates(&conn, crypto, time_ago, 900)? + self.get_price_from_aggregates(crypto, time_ago as i64, 900).await? }; - // Only add the change if we have data for that time period if let Some(price) = old_price { debug!( "Found {} {} price: ${} (current: ${})", @@ -220,7 +181,7 @@ impl PriceDatabase { changes.push(format!( "{} {}{:.2}% ({})", arrow, sign, change_percent, label - )); + ))); } else { debug!( "No {} {} price data found for time_ago: {}", @@ -243,83 +204,57 @@ impl PriceDatabase { } } - /// Get price from raw data table - fn get_price_from_raw_data( - &self, - conn: &Connection, - crypto: &str, - time_ago: u64, - ) -> BotResult> { - let mut stmt = conn.prepare_cached( - "SELECT price FROM prices WHERE crypto_name = ? AND timestamp >= ? ORDER BY timestamp ASC LIMIT 1" - )?; - - let rows = stmt.query_map([crypto, &time_ago.to_string()], |row| Ok(row.get(0)?))?; - - let mut prices = rows.collect::, _>>()?; - Ok(prices.pop()) + async fn get_price_from_raw_data(&self, crypto: &str, time_ago: i64) -> BotResult> { + let row: Option<(f64,)> = sqlx::query_as( + "SELECT price FROM prices WHERE crypto_name = $1 AND timestamp >= $2 ORDER BY timestamp ASC LIMIT 1", + ) + .bind(crypto) + .bind(time_ago) + .fetch_optional(&self.pool) + .await + .map_err(|e| BotError::Database(e.to_string()))?; + + Ok(row.map(|r| r.0)) } - /// Get price from aggregated data table - fn get_price_from_aggregates( - &self, - conn: &Connection, - crypto: &str, - time_ago: u64, - bucket_duration: u64, - ) -> BotResult> { - // Find the bucket that contains or is closest to the target time - // We want the bucket where bucket_start <= time_ago < bucket_start + bucket_duration - // Or the closest bucket if no exact match - let mut stmt = conn.prepare_cached( + async fn get_price_from_aggregates(&self, crypto: &str, time_ago: i64, bucket_duration: i64) -> BotResult> { + let row: Option<(f64,)> = sqlx::query_as( "SELECT open_price FROM price_aggregates - WHERE crypto_name = ? AND bucket_duration = ? - AND bucket_start <= ? + WHERE crypto_name = $1 AND bucket_duration = $2 + AND bucket_start <= $3 ORDER BY bucket_start DESC LIMIT 1", - )?; - - let rows = stmt.query_map( - [crypto, &bucket_duration.to_string(), &time_ago.to_string()], - |row| Ok(row.get(0)?), - )?; - - let mut prices = rows.collect::, _>>()?; - Ok(prices.pop()) + ) + .bind(crypto) + .bind(bucket_duration) + .bind(time_ago) + .fetch_optional(&self.pool) + .await + .map_err(|e| BotError::Database(e.to_string()))?; + + Ok(row.map(|r| r.0)) } - /// Get price indicator from database for status display - pub fn get_price_indicator(&self, crypto_name: &str, current_price: f64) -> (String, f64) { + pub async fn get_price_indicator(&self, crypto_name: &str, current_price: f64) -> (String, f64) { let current_time = match get_current_timestamp() { - Ok(time) => time, + Ok(time) => time as i64, Err(_) => return ("๐Ÿ”„".to_string(), 0.0), }; - let conn = match self.get_connection() { - Ok(conn) => conn, - Err(_) => return ("๐Ÿ”„".to_string(), 0.0), - }; - - let mut stmt = match conn.prepare_cached( - "SELECT price FROM prices WHERE crypto_name = ? AND timestamp >= ? ORDER BY timestamp ASC LIMIT 1" - ) { - Ok(stmt) => stmt, - Err(_) => return ("๐Ÿ”„".to_string(), 0.0), - }; - - let one_hour_ago = current_time - 3600; // 1 hour - let rows = match stmt.query_map([crypto_name, &one_hour_ago.to_string()], |row| { - Ok(row.get(0)?) - }) { - Ok(rows) => rows, - Err(_) => return ("๐Ÿ”„".to_string(), 0.0), - }; - - let mut prices = match rows.collect::, _>>() { - Ok(prices) => prices, - Err(_) => return ("๐Ÿ”„".to_string(), 0.0), - }; - - if let Some(oldest_price) = prices.pop() { + let one_hour_ago = current_time - 3600; + + let row: Option<(f64,)> = sqlx::query_as( + "SELECT price FROM prices WHERE crypto_name = $1 AND timestamp >= $2 ORDER BY timestamp ASC LIMIT 1", + ) + .bind(crypto_name) + .bind(one_hour_ago) + .fetch_optional(&self.pool) + .await + .map_err(|e| { + debug!("Failed to get price indicator: {}", e); + BotError::Database(e.to_string()) + }).ok().flatten(); + + if let Some((oldest_price,)) = row { match calculate_percentage_change(current_price, oldest_price) { Ok(change_percent) => { let arrow = get_change_arrow(change_percent); @@ -329,50 +264,36 @@ impl PriceDatabase { } } - // No history yet ("๐Ÿ”„".to_string(), 0.0) } - /// Get price history for charting (up to specified days) - /// Returns vector of (timestamp, price) tuples - pub fn get_price_history(&self, crypto_name: &str, days: u64) -> BotResult> { - let conn = self.get_connection()?; - let current_time = get_current_timestamp()?; - let start_time = current_time - (days * 86400); - - // Strategy: Combine aggregated history + Recent raw data - // 1. Fetch best available aggregates - // 2. Fetch raw data that is newer than the newest aggregate - // 3. Merge and sort + pub async fn get_price_history(&self, crypto_name: &str, days: u64) -> BotResult> { + let current_time = get_current_timestamp()? as i64; + let start_time = current_time - (days as i64 * 86400); let mut history = Vec::new(); - let mut last_aggregated_time = start_time as i64; + let mut last_aggregated_time = start_time; - // 1. Fetch Aggregates - // Try to get 5-minute buckets first, then 1-minute (fallback), then 15m, then 1h - // This ensures we get the best resolution available for the time range let bucket_durations = vec![300, 60, 900, 3600]; for duration in bucket_durations { - let mut stmt = conn.prepare_cached( + let rows: Vec<(i64, f64)> = sqlx::query_as( "SELECT bucket_start, open_price FROM price_aggregates - WHERE crypto_name = ? AND bucket_duration = ? AND bucket_start >= ? + WHERE crypto_name = $1 AND bucket_duration = $2 AND bucket_start >= $3 ORDER BY bucket_start ASC", - )?; - - let rows = stmt.query_map( - [crypto_name, &duration.to_string(), &start_time.to_string()], - |row| Ok((row.get::<_, i64>(0)?, row.get::<_, f64>(1)?)), - )?; - - let data: Vec<(i64, f64)> = rows.collect::, _>>()?; - - if !data.is_empty() { - // If we found data, record the last timestamp so we know where to start raw data - if let Some((ts, _)) = data.last() { + ) + .bind(crypto_name) + .bind(duration) + .bind(start_time) + .fetch_all(&self.pool) + .await + .map_err(|e| BotError::Database(e.to_string()))?; + + if !rows.is_empty() { + if let Some((ts, _)) = rows.last() { last_aggregated_time = *ts; } - history = data; + history = rows; debug!( "Found {} aggregated points for {} using {}-second buckets", history.len(), @@ -383,79 +304,66 @@ impl PriceDatabase { } } - // 2. Fetch Raw Data (Newer than last aggregate) - // This covers the gap from the last cleanup/aggregation run to NOW - // Also helps if no aggregates exist at all (last_aggregated_time == start_time) - debug!( "Fetching raw prices for {} newer than {}", crypto_name, last_aggregated_time ); - let mut stmt = conn.prepare_cached( + let rows: Vec<(i64, f64)> = sqlx::query_as( "SELECT timestamp, price FROM prices - WHERE crypto_name = ? AND timestamp > ? + WHERE crypto_name = $1 AND timestamp > $2 ORDER BY timestamp ASC", - )?; - - let rows = stmt.query_map([crypto_name, &last_aggregated_time.to_string()], |row| { - Ok((row.get::<_, i64>(0)?, row.get::<_, f64>(1)?)) - })?; - - let raw_data: Vec<(i64, f64)> = rows.collect::, _>>()?; - - // Downsample raw data if there's too much (e.g., if we have no aggregates and 30 days of raw data) - // But typically this will just be the last 24h ~ few hundred points max - if raw_data.len() > 1000 { - let step = raw_data.len() / 500; - let downsampled = raw_data + ) + .bind(crypto_name) + .bind(last_aggregated_time) + .fetch_all(&self.pool) + .await + .map_err(|e| BotError::Database(e.to_string()))?; + + if rows.len() > 1000 { + let step = rows.len() / 500; + let downsampled = rows .into_iter() .enumerate() .filter(|(i, _)| i % step == 0) .map(|(_, val)| val); history.extend(downsampled); } else { - history.extend(raw_data); + history.extend(rows); } - // 3. Final Sort (just in case, though append should be sorted) history.sort_by_key(|k| k.0); Ok(history) } - /// Clean up old price records from the database - pub fn cleanup_old_prices(&self) -> BotResult<()> { - let conn = self.get_connection()?; - - // Keep only the last 60 days of data + pub async fn cleanup_old_prices(&self) -> BotResult<()> { let cutoff_time = get_current_timestamp()? - (PRICE_HISTORY_DAYS * 24 * 3600); - let deleted = conn.execute( - "DELETE FROM prices WHERE timestamp < ?", - [&cutoff_time.to_string()], - )?; + let result = sqlx::query("DELETE FROM prices WHERE timestamp < $1") + .bind(cutoff_time as i64) + .execute(&self.pool) + .await + .map_err(|e| BotError::Database(e.to_string()))?; - if deleted > 0 { - info!("Cleaned up {} old price records from database", deleted); + if result.rows_affected() > 0 { + info!("Cleaned up {} old price records from database", result.rows_affected()); } Ok(()) } - /// Perform periodic cleanup if needed - pub fn maybe_cleanup(&self) { + pub async fn maybe_cleanup(&self) { static LAST_CLEANUP: AtomicU64 = AtomicU64::new(0); if let Ok(current_time) = get_current_timestamp() { let last_cleanup = LAST_CLEANUP.load(Ordering::Relaxed); if current_time - last_cleanup > CLEANUP_INTERVAL_SECONDS { - match self.cleanup_old_prices() { - Ok(_) => debug!("Database cleanup completed"), - Err(e) => error!("Failed to cleanup old prices: {}", e), + if let Err(e) = self.cleanup_old_prices().await { + error!("Failed to cleanup old prices: {}", e); } LAST_CLEANUP.store(current_time, Ordering::Relaxed); } } } -} +} \ No newline at end of file diff --git a/src/database_tests.rs b/src/database_tests.rs index 160074a..ca2331b 100644 --- a/src/database_tests.rs +++ b/src/database_tests.rs @@ -1,46 +1,52 @@ #[cfg(test)] mod tests { use crate::PriceDatabase; - use tempfile::TempDir; - fn setup_temp_db() -> (TempDir, PriceDatabase) { - let temp_dir = TempDir::new().expect("Failed to create temp dir"); - let db_path = temp_dir.path().join("test.db"); - let db = PriceDatabase::new(db_path.to_str().unwrap()).expect("Failed to create database"); - (temp_dir, db) + fn get_test_database_url() -> String { + std::env::var("TEST_DATABASE_URL").unwrap_or_else(|_| { + "postgres://postgres:postgres@localhost:5432/pricebot_test".to_string() + }) } - #[test] - fn test_database_initializes() { - let (_temp_dir, db) = setup_temp_db(); - // Just verify database can be created and queried - assert!(db.get_all_latest_prices().is_ok()); + #[tokio::test] + #[ignore] + async fn test_database_initializes() { + let db_url = get_test_database_url(); + let db = PriceDatabase::new(&db_url) + .await + .expect("Failed to create database"); + let result = db.get_all_latest_prices().await; + assert!(result.is_ok()); } - #[test] - fn test_save_and_retrieve_price() { - let (_temp_dir, db) = setup_temp_db(); + #[tokio::test] + #[ignore] + async fn test_save_and_retrieve_price() { + let db_url = get_test_database_url(); + let db = PriceDatabase::new(&db_url) + .await + .expect("Failed to create database"); - // Save a price - let result = db.save_price("BTC", 50000.0); + let result = db.save_price("BTC", 50000.0).await; assert!(result.is_ok()); - // Retrieve it - let price = db.get_latest_price("BTC"); + let price = db.get_latest_price("BTC").await; assert!(price.is_ok()); assert_eq!(price.unwrap(), 50000.0); } - #[test] - fn test_save_invalid_price_rejected() { - let (_temp_dir, db) = setup_temp_db(); + #[tokio::test] + #[ignore] + async fn test_save_invalid_price_rejected() { + let db_url = get_test_database_url(); + let db = PriceDatabase::new(&db_url) + .await + .expect("Failed to create database"); - // Zero price should be skipped (not saved) - let result = db.save_price("BTC", 0.0); - assert!(result.is_ok()); // Returns ok but doesn't save zero + let result = db.save_price("BTC", 0.0).await; + assert!(result.is_ok()); - // Negative price should be skipped - let result = db.save_price("BTC", -100.0); - assert!(result.is_ok()); // Returns ok but doesn't save negative + let result = db.save_price("BTC", -100.0).await; + assert!(result.is_ok()); } } diff --git a/src/db_cleanup.rs b/src/db_cleanup.rs index 52c786a..d18b623 100644 --- a/src/db_cleanup.rs +++ b/src/db_cleanup.rs @@ -5,92 +5,44 @@ use crate::config::{ use crate::database::PriceDatabase; use crate::errors::{BotError, BotResult}; use crate::health::HealthState; - -use rusqlite::Connection; +use sqlx::PgPool; use std::sync::Arc; use std::time::Duration; use tokio::time::sleep; use tracing::{debug, error, info, warn}; -/// Database cleanup service for aggregating and compacting price data pub struct DatabaseCleanup { health: Arc, - database: Arc, + pool: PgPool, } impl DatabaseCleanup { - pub fn new(database: Arc) -> Self { + pub fn new(database: &Arc) -> Self { let health = Arc::new(HealthState::new("DB-CLEANUP".to_string())); - Self { health, database } - } - - /// Get a database connection from the pool - fn get_connection( - &self, - ) -> BotResult> { - self.database.get_connection() - } - - /// Initialize the aggregated data table - fn init_aggregated_table(&self) -> BotResult<()> { - let conn = self.get_connection()?; - - conn.execute( - "CREATE TABLE IF NOT EXISTS price_aggregates ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - crypto_name TEXT NOT NULL, - bucket_start INTEGER NOT NULL, - bucket_duration INTEGER NOT NULL, - open_price REAL NOT NULL, - high_price REAL NOT NULL, - low_price REAL NOT NULL, - close_price REAL NOT NULL, - avg_price REAL NOT NULL, - sample_count INTEGER NOT NULL, - created_at DATETIME DEFAULT CURRENT_TIMESTAMP - )", - [], - )?; - - // Create indexes for efficient queries - conn.execute( - "CREATE INDEX IF NOT EXISTS idx_aggregates_crypto_bucket - ON price_aggregates(crypto_name, bucket_start, bucket_duration)", - [], - )?; - - conn.execute( - "CREATE INDEX IF NOT EXISTS idx_prices_crypto_timestamp - ON prices(crypto_name, timestamp)", - [], - )?; - - info!("โœ… Initialized aggregated data table and indexes"); - Ok(()) + Self { + health, + pool: database.pool(), + } } - /// Aggregate raw data into time buckets with batching to reduce lock time - fn aggregate_data( + async fn aggregate_data( &self, - bucket_duration_seconds: u64, - older_than_seconds: u64, + bucket_duration_seconds: i64, + older_than_seconds: i64, ) -> BotResult { info!( " ๐Ÿ” Checking for data older than {} seconds to aggregate into {}-second buckets", older_than_seconds, bucket_duration_seconds ); - let conn = self.get_connection()?; let current_time = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map_err(|e| BotError::SystemTime(format!("System time error: {}", e)))? - .as_secs(); + .as_secs() as i64; let cutoff_time = current_time - older_than_seconds; - let bucket_duration = bucket_duration_seconds as i64; - // Process in smaller batches to reduce lock contention - let batch_size = 100; + let batch_size: i64 = 100; let mut total_aggregated = 0u64; let mut batch_number = 0; @@ -101,106 +53,89 @@ impl DatabaseCleanup { batch_number, bucket_duration_seconds ); - // Get a small batch of data to aggregate - let mut stmt = conn.prepare( - "SELECT crypto_name, - (timestamp / ?) * ? as bucket_start, - MIN(price) as low_price, - MAX(price) as high_price, - AVG(price) as avg_price, - COUNT(*) as sample_count - FROM prices - WHERE timestamp < ? - AND NOT EXISTS ( - SELECT 1 FROM price_aggregates pa - WHERE pa.crypto_name = prices.crypto_name - AND pa.bucket_start = (prices.timestamp / ?) * ? - AND pa.bucket_duration = ? - ) - GROUP BY crypto_name, bucket_start - HAVING COUNT(*) > 0 - ORDER BY crypto_name, bucket_start - LIMIT ?", - )?; - - let rows = stmt.query_map( - [ - bucket_duration, - bucket_duration, // bucket_start calculation - cutoff_time as i64, // WHERE timestamp < cutoff - bucket_duration, - bucket_duration, - bucket_duration, // NOT EXISTS check - batch_size as i64, // LIMIT - ], - |row| { - Ok(( - row.get::<_, String>(0)?, // crypto_name - row.get::<_, i64>(1)?, // bucket_start - row.get::<_, f64>(2)?, // low_price - row.get::<_, f64>(3)?, // high_price - row.get::<_, f64>(4)?, // avg_price - row.get::<_, i64>(5)?, // sample_count - )) - }, - )?; - - let batch_data: Vec<_> = rows.collect::, _>>()?; - - if batch_data.is_empty() { + let rows: Vec<(String, i64, f64, f64, f64, i64)> = sqlx::query_as( + r#" + SELECT crypto_name, + (timestamp / $1) * $1 as bucket_start, + MIN(price) as low_price, + MAX(price) as high_price, + AVG(price) as avg_price, + COUNT(*) as sample_count + FROM prices + WHERE timestamp < $2 + AND NOT EXISTS ( + SELECT 1 FROM price_aggregates pa + WHERE pa.crypto_name = prices.crypto_name + AND pa.bucket_start = (prices.timestamp / $1) * $1 + AND pa.bucket_duration = $1 + ) + GROUP BY crypto_name, bucket_start + HAVING COUNT(*) > 0 + ORDER BY crypto_name, bucket_start + LIMIT $3 + "#, + ) + .bind(bucket_duration_seconds) + .bind(cutoff_time) + .bind(batch_size) + .fetch_all(&self.pool) + .await + .map_err(|e| BotError::Database(e.to_string()))?; + + if rows.is_empty() { debug!( " โœ… No more data to aggregate for {}-second buckets", bucket_duration_seconds ); - break; // No more data to process + break; } debug!( " ๐Ÿ“Š Found {} records to aggregate in batch {}", - batch_data.len(), + rows.len(), batch_number ); - // Process this batch in a transaction - let tx = conn.unchecked_transaction()?; let mut batch_count = 0u64; - for (crypto_name, bucket_start, low_price, high_price, avg_price, sample_count) in - batch_data + for (crypto_name, bucket_start, low_price, high_price, avg_price, sample_count) in rows { - // Get open and close prices separately for accuracy - let open_price = - self.get_bucket_open_price(&conn, &crypto_name, bucket_start, bucket_duration)?; - let close_price = self.get_bucket_close_price( - &conn, - &crypto_name, - bucket_start, - bucket_duration, - )?; - - // Insert the aggregated data - tx.execute( - "INSERT INTO price_aggregates - (crypto_name, bucket_start, bucket_duration, open_price, high_price, low_price, close_price, avg_price, sample_count) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", - [ - &crypto_name, - &bucket_start.to_string(), - &bucket_duration.to_string(), - &open_price.to_string(), - &high_price.to_string(), - &low_price.to_string(), - &close_price.to_string(), - &avg_price.to_string(), - &sample_count.to_string(), - ] - )?; + let open_price = self + .get_bucket_open_price( + crypto_name.clone(), + bucket_start, + bucket_duration_seconds, + ) + .await?; + let close_price = self + .get_bucket_close_price( + crypto_name.clone(), + bucket_start, + bucket_duration_seconds, + ) + .await?; + + sqlx::query( + r#"INSERT INTO price_aggregates + (crypto_name, bucket_start, bucket_duration, open_price, high_price, low_price, close_price, avg_price, sample_count) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)"#, + ) + .bind(&crypto_name) + .bind(bucket_start) + .bind(bucket_duration_seconds) + .bind(open_price) + .bind(high_price) + .bind(low_price) + .bind(close_price) + .bind(avg_price) + .bind(sample_count) + .execute(&self.pool) + .await + .map_err(|e| BotError::Database(e.to_string()))?; batch_count += 1; } - // Commit this batch - tx.commit()?; total_aggregated += batch_count; debug!( @@ -208,8 +143,7 @@ impl DatabaseCleanup { batch_number, batch_count, total_aggregated ); - // Small delay between batches to allow other processes to access DB - std::thread::sleep(std::time::Duration::from_millis(100)); + sleep(Duration::from_millis(100)).await; } if total_aggregated > 0 { @@ -222,162 +156,154 @@ impl DatabaseCleanup { Ok(total_aggregated) } - /// Get the opening price for a bucket - fn get_bucket_open_price( + async fn get_bucket_open_price( &self, - conn: &Connection, - crypto_name: &str, + crypto_name: String, bucket_start: i64, bucket_duration: i64, ) -> BotResult { let bucket_end = bucket_start + bucket_duration; - let mut stmt = conn.prepare( + + let row: (f64,) = sqlx::query_as( "SELECT price FROM prices - WHERE crypto_name = ? AND timestamp >= ? AND timestamp < ? + WHERE crypto_name = $1 AND timestamp >= $2 AND timestamp < $3 ORDER BY timestamp ASC LIMIT 1", - )?; - - let price: f64 = stmt.query_row( - [ - crypto_name, - &bucket_start.to_string(), - &bucket_end.to_string(), - ], - |row| row.get(0), - )?; - - Ok(price) + ) + .bind(&crypto_name) + .bind(bucket_start) + .bind(bucket_end) + .fetch_one(&self.pool) + .await + .map_err(|e| BotError::Database(e.to_string()))?; + + Ok(row.0) } - /// Get the closing price for a bucket - fn get_bucket_close_price( + async fn get_bucket_close_price( &self, - conn: &Connection, - crypto_name: &str, + crypto_name: String, bucket_start: i64, bucket_duration: i64, ) -> BotResult { let bucket_end = bucket_start + bucket_duration; - let mut stmt = conn.prepare( + + let row: (f64,) = sqlx::query_as( "SELECT price FROM prices - WHERE crypto_name = ? AND timestamp >= ? AND timestamp < ? + WHERE crypto_name = $1 AND timestamp >= $2 AND timestamp < $3 ORDER BY timestamp DESC LIMIT 1", - )?; - - let price: f64 = stmt.query_row( - [ - crypto_name, - &bucket_start.to_string(), - &bucket_end.to_string(), - ], - |row| row.get(0), - )?; - - Ok(price) + ) + .bind(&crypto_name) + .bind(bucket_start) + .bind(bucket_end) + .fetch_one(&self.pool) + .await + .map_err(|e| BotError::Database(e.to_string()))?; + + Ok(row.0) } - /// Delete raw data that has been successfully aggregated - fn cleanup_aggregated_raw_data(&self, older_than_seconds: u64) -> BotResult { + async fn cleanup_aggregated_raw_data(&self, older_than_seconds: i64) -> BotResult { info!( " ๐Ÿ” Checking how many raw records are older than {} seconds", older_than_seconds ); - let conn = self.get_connection()?; let current_time = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map_err(|e| BotError::SystemTime(format!("System time error: {}", e)))? - .as_secs(); + .as_secs() as i64; let cutoff_time = current_time - older_than_seconds; - // First, count how many records we're about to delete - let mut count_stmt = conn.prepare("SELECT COUNT(*) FROM prices WHERE timestamp < ?")?; - let count: i64 = count_stmt.query_row([cutoff_time as i64], |row| row.get(0))?; + let count: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM prices WHERE timestamp < $1") + .bind(cutoff_time) + .fetch_one(&self.pool) + .await + .map_err(|e| BotError::Database(e.to_string()))?; info!( " ๐Ÿ“Š Found {} raw records older than {} seconds", - count, older_than_seconds + count.0, older_than_seconds ); - if count == 0 { + if count.0 == 0 { info!(" โœ… No old raw data to clean up"); return Ok(0); } info!(" ๐Ÿ—‘๏ธ Deleting old raw records in batches..."); - // Delete in batches to avoid hanging let mut total_deleted = 0i64; - let batch_size = 10000; + let batch_size: i64 = 10000; loop { - // SQLite doesn't support DELETE ... LIMIT until 3.35.0 - // So we select rowids first, then delete - let deleted = conn.execute( - "DELETE FROM prices - WHERE rowid IN ( - SELECT rowid FROM prices - WHERE timestamp < ? + let result = sqlx::query( + r#"DELETE FROM prices + WHERE id IN ( + SELECT p.id FROM prices p + WHERE p.timestamp < $1 AND EXISTS ( SELECT 1 FROM price_aggregates pa - WHERE pa.crypto_name = prices.crypto_name - AND pa.bucket_start <= prices.timestamp - AND pa.bucket_start + pa.bucket_duration > prices.timestamp + WHERE pa.crypto_name = p.crypto_name + AND pa.bucket_start <= p.timestamp + AND pa.bucket_start + pa.bucket_duration > p.timestamp ) - LIMIT ? - )", - rusqlite::params![cutoff_time as i64, batch_size as i64], - )?; - + LIMIT $2 + )"#, + ) + .bind(cutoff_time) + .bind(batch_size) + .execute(&self.pool) + .await + .map_err(|e| BotError::Database(e.to_string()))?; + + let deleted = result.rows_affected() as i64; if deleted == 0 { break; } - total_deleted += deleted as i64; + total_deleted += deleted; info!(" Deleted {} records (total: {})", deleted, total_deleted); } info!( - " {} raw price records โœ… Successfully deleted older than {} seconds", + " {} raw price records โœ… Successfully deleted older than {} seconds", total_deleted, older_than_seconds ); Ok(total_deleted as u64) } - /// Delete old aggregated data beyond retention period - fn cleanup_old_aggregates( + async fn cleanup_old_aggregates( &self, - bucket_duration_seconds: u64, - older_than_seconds: u64, + bucket_duration_seconds: i64, + older_than_seconds: i64, ) -> BotResult { info!( " ๐Ÿงน Cleaning up {}-second aggregates older than {} seconds", bucket_duration_seconds, older_than_seconds ); - let conn = self.get_connection()?; let current_time = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map_err(|e| BotError::SystemTime(format!("System time error: {}", e)))? - .as_secs(); + .as_secs() as i64; let cutoff_time = current_time - older_than_seconds; - // First count what we're about to delete - let mut count_stmt = conn.prepare( - "SELECT COUNT(*) FROM price_aggregates WHERE bucket_start < ? AND bucket_duration = ?", - )?; - let count: i64 = count_stmt.query_row( - [cutoff_time as i64, bucket_duration_seconds as i64], - |row| row.get(0), - )?; + let count: (i64,) = sqlx::query_as( + "SELECT COUNT(*) FROM price_aggregates WHERE bucket_start < $1 AND bucket_duration = $2", + ) + .bind(cutoff_time) + .bind(bucket_duration_seconds) + .fetch_one(&self.pool) + .await + .map_err(|e| BotError::Database(e.to_string()))?; - if count > 0 { + if count.0 > 0 { info!( " ๐Ÿ—‘๏ธ Deleting {} old {}-second aggregate records...", - count, bucket_duration_seconds + count.0, bucket_duration_seconds ); } else { info!( @@ -386,258 +312,186 @@ impl DatabaseCleanup { ); } - let deleted = conn.execute( - "DELETE FROM price_aggregates - WHERE bucket_start < ? AND bucket_duration = ?", - [cutoff_time as i64, bucket_duration_seconds as i64], - )?; + let result = sqlx::query( + "DELETE FROM price_aggregates WHERE bucket_start < $1 AND bucket_duration = $2", + ) + .bind(cutoff_time) + .bind(bucket_duration_seconds) + .execute(&self.pool) + .await + .map_err(|e| BotError::Database(e.to_string()))?; - if deleted > 0 { + if result.rows_affected() > 0 { info!( " โœ… Deleted {} aggregated records ({}-second buckets)", - deleted, bucket_duration_seconds + result.rows_affected(), + bucket_duration_seconds ); } - Ok(deleted as u64) - } - - /// Vacuum the database to reclaim space - fn vacuum_database(&self) -> BotResult<()> { - let conn = self.get_connection()?; - - info!("๐Ÿงน Starting database vacuum..."); - conn.execute("VACUUM", [])?; - info!("โœ… Database vacuum completed"); - - Ok(()) + Ok(result.rows_affected()) } - /// Get database statistics - fn get_database_stats(&self) -> BotResult<()> { - let conn = self.get_connection()?; - - // Count raw price records - let raw_count: i64 = - conn.query_row("SELECT COUNT(*) FROM prices", [], |row: &rusqlite::Row| { - row.get(0) - })?; - - // Count aggregated records by bucket size - let mut stmt = conn.prepare( - "SELECT bucket_duration, COUNT(*) FROM price_aggregates GROUP BY bucket_duration ORDER BY bucket_duration" - )?; + async fn get_database_stats(&self) -> BotResult<()> { + let raw_count: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM prices") + .fetch_one(&self.pool) + .await + .map_err(|e| BotError::Database(e.to_string()))?; - let rows = stmt.query_map([], |row: &rusqlite::Row| { - Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?)) - })?; + let aggregates: Vec<(i64, i64)> = sqlx::query_as( + "SELECT bucket_duration, COUNT(*) FROM price_aggregates GROUP BY bucket_duration ORDER BY bucket_duration", + ) + .fetch_all(&self.pool) + .await + .map_err(|e| BotError::Database(e.to_string()))?; info!("๐Ÿ“Š Database Statistics:"); - info!(" Raw price records: {}", raw_count); + info!(" Raw price records: {}", raw_count.0); - for row in rows { - let (duration, count) = row?; + for (duration, count) in aggregates { info!(" {}-second aggregates: {}", duration, count); } Ok(()) } - /// Perform complete cleanup cycle with retry logic - pub async fn perform_cleanup(&self) -> BotResult<()> { - const MAX_RETRIES: u32 = 3; - - for attempt in 1..=MAX_RETRIES { - match self.perform_cleanup_attempt().await { - Ok(()) => return Ok(()), - Err(e) => { - error!("โŒ Cleanup attempt {} failed: {}", attempt, e); - if attempt < MAX_RETRIES { - info!("โณ Retrying cleanup in 30 seconds..."); - tokio::time::sleep(Duration::from_secs(30)).await; - } else { - return Err(e); - } - } - } - } - unreachable!() - } - - /// Aggregate data from smaller buckets into larger buckets (e.g., 1m -> 5m) - fn aggregate_buckets( + async fn aggregate_buckets( &self, - source_duration: u64, - target_duration: u64, - older_than_seconds: u64, + source_duration: i64, + target_duration: i64, + older_than_seconds: i64, ) -> BotResult { info!( " ๐Ÿ” Aggregating {}-second buckets older than {} seconds into {}-second buckets", source_duration, older_than_seconds, target_duration ); - let conn = self.get_connection()?; let current_time = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map_err(|e| BotError::SystemTime(format!("System time error: {}", e)))? - .as_secs(); + .as_secs() as i64; let cutoff_time = current_time - older_than_seconds; - // Process in batches - let batch_size = 100; + let batch_size: i64 = 100; let mut total_aggregated = 0u64; let mut batch_number = 0; loop { batch_number += 1; - // Get a batch of source buckets to aggregate - // We group by the NEW bucket start time - let mut stmt = conn.prepare( - "SELECT crypto_name, - (bucket_start / ?) * ? as new_bucket_start, - MIN(low_price) as low_price, - MAX(high_price) as high_price, - SUM(avg_price * sample_count) / SUM(sample_count) as avg_price, - SUM(sample_count) as sample_count - FROM price_aggregates - WHERE bucket_duration = ? - AND bucket_start < ? - AND NOT EXISTS ( - SELECT 1 FROM price_aggregates pa - WHERE pa.crypto_name = price_aggregates.crypto_name - AND pa.bucket_start = (price_aggregates.bucket_start / ?) * ? - AND pa.bucket_duration = ? - ) - GROUP BY crypto_name, new_bucket_start - HAVING COUNT(*) > 0 - ORDER BY crypto_name, new_bucket_start - LIMIT ?", - )?; - - let rows = stmt.query_map( - [ - target_duration, - target_duration, // new_bucket_start calculation - source_duration, // WHERE bucket_duration = source - cutoff_time, // AND bucket_start < cutoff - target_duration, - target_duration, - target_duration, // NOT EXISTS check - batch_size as u64, // LIMIT - ], - |row| { - Ok(( - row.get::<_, String>(0)?, // crypto_name - row.get::<_, i64>(1)?, // new_bucket_start - row.get::<_, f64>(2)?, // low_price - row.get::<_, f64>(3)?, // high_price - row.get::<_, f64>(4)?, // avg_price - row.get::<_, i64>(5)?, // sample_count - )) - }, - )?; - - let batch_data: Vec<_> = rows.collect::, _>>()?; - - if batch_data.is_empty() { - break; // No more data to process + let rows: Vec<(String, i64, f64, f64, f64, i64)> = sqlx::query_as( + r#" + SELECT crypto_name, + (bucket_start / $1) * $1 as new_bucket_start, + MIN(low_price) as low_price, + MAX(high_price) as high_price, + SUM(avg_price * sample_count) / SUM(sample_count) as avg_price, + SUM(sample_count) as sample_count + FROM price_aggregates + WHERE bucket_duration = $2 + AND bucket_start < $3 + AND NOT EXISTS ( + SELECT 1 FROM price_aggregates pa + WHERE pa.crypto_name = price_aggregates.crypto_name + AND pa.bucket_start = (price_aggregates.bucket_start / $1) * $1 + AND pa.bucket_duration = $1 + ) + GROUP BY crypto_name, new_bucket_start + HAVING COUNT(*) > 0 + ORDER BY crypto_name, new_bucket_start + LIMIT $4 + "#, + ) + .bind(target_duration) + .bind(source_duration) + .bind(cutoff_time) + .bind(batch_size) + .fetch_all(&self.pool) + .await + .map_err(|e| BotError::Database(e.to_string()))?; + + if rows.is_empty() { + break; } debug!( " ๐Ÿ“Š Found {} bucket groups to aggregate in batch {}", - batch_data.len(), + rows.len(), batch_number ); - // Process this batch in a transaction - let tx = conn.unchecked_transaction()?; let mut batch_count = 0u64; - for (crypto_name, bucket_start, low_price, high_price, avg_price, sample_count) in - batch_data + for (crypto_name, bucket_start, low_price, high_price, avg_price, sample_count) in rows { - // For open/close, we need to query the source buckets - // Open price = Open price of the earliest source bucket in this range - // Close price = Close price of the latest source bucket in this range - let bucket_end = bucket_start + target_duration as i64; + let bucket_end = bucket_start + target_duration; - // Get open price - let open_price: f64 = match tx.query_row( + let open_price: f64 = match sqlx::query_as( "SELECT open_price FROM price_aggregates - WHERE crypto_name = ? AND bucket_duration = ? - AND bucket_start >= ? AND bucket_start < ? + WHERE crypto_name = $1 AND bucket_duration = $2 + AND bucket_start >= $3 AND bucket_start < $4 ORDER BY bucket_start ASC LIMIT 1", - [ - &crypto_name, - &source_duration.to_string(), - &bucket_start.to_string(), - &bucket_end.to_string(), - ], - |row| row.get(0), - ) { - Ok(price) => price, - Err(e) => { - warn!( - "Failed to get open price for {}: {}, using avg", - crypto_name, e - ); + ) + .bind(&crypto_name) + .bind(source_duration) + .bind(bucket_start) + .bind(bucket_end) + .fetch_optional(&self.pool) + .await + .map_err(|e| BotError::Database(e.to_string()))? + { + Some((price,)) => price, + None => { + warn!("Failed to get open price for {}, using avg", crypto_name); avg_price } }; - // Get close price - let close_price: f64 = match tx.query_row( + let close_price: f64 = match sqlx::query_as( "SELECT close_price FROM price_aggregates - WHERE crypto_name = ? AND bucket_duration = ? - AND bucket_start >= ? AND bucket_start < ? + WHERE crypto_name = $1 AND bucket_duration = $2 + AND bucket_start >= $3 AND bucket_start < $4 ORDER BY bucket_start DESC LIMIT 1", - [ - &crypto_name, - &source_duration.to_string(), - &bucket_start.to_string(), - &bucket_end.to_string(), - ], - |row| row.get(0), - ) { - Ok(price) => price, - Err(e) => { - warn!( - "Failed to get close price for {}: {}, using avg", - crypto_name, e - ); + ) + .bind(&crypto_name) + .bind(source_duration) + .bind(bucket_start) + .bind(bucket_end) + .fetch_optional(&self.pool) + .await + .map_err(|e| BotError::Database(e.to_string()))? + { + Some((price,)) => price, + None => { + warn!("Failed to get close price for {}, using avg", crypto_name); avg_price } }; - // Insert the aggregated data - tx.execute( - "INSERT INTO price_aggregates - (crypto_name, bucket_start, bucket_duration, open_price, high_price, low_price, close_price, avg_price, sample_count) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", - [ - &crypto_name, - &bucket_start.to_string(), - &target_duration.to_string(), - &open_price.to_string(), - &high_price.to_string(), - &low_price.to_string(), - &close_price.to_string(), - &avg_price.to_string(), - &sample_count.to_string(), - ] - )?; + sqlx::query( + r#"INSERT INTO price_aggregates + (crypto_name, bucket_start, bucket_duration, open_price, high_price, low_price, close_price, avg_price, sample_count) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)"#, + ) + .bind(&crypto_name) + .bind(bucket_start) + .bind(target_duration) + .bind(open_price) + .bind(high_price) + .bind(low_price) + .bind(close_price) + .bind(avg_price) + .bind(sample_count) + .execute(&self.pool) + .await + .map_err(|e| BotError::Database(e.to_string()))?; batch_count += 1; } - // Commit this batch - tx.commit()?; total_aggregated += batch_count; - - // Small delay - std::thread::sleep(std::time::Duration::from_millis(50)); + sleep(Duration::from_millis(50)).await; } if total_aggregated > 0 { @@ -650,51 +504,67 @@ impl DatabaseCleanup { Ok(total_aggregated) } - /// Single cleanup attempt + pub async fn perform_cleanup(&self) -> BotResult<()> { + const MAX_RETRIES: u32 = 3; + + for attempt in 1..=MAX_RETRIES { + match self.perform_cleanup_attempt().await { + Ok(()) => return Ok(()), + Err(e) => { + error!("โŒ Cleanup attempt {} failed: {}", attempt, e); + if attempt < MAX_RETRIES { + info!("โณ Retrying cleanup in 30 seconds..."); + tokio::time::sleep(Duration::from_secs(30)).await; + } else { + return Err(e); + } + } + } + } + unreachable!() + } + async fn perform_cleanup_attempt(&self) -> BotResult<()> { info!("๐Ÿงน Starting database cleanup cycle..."); - self.health.update_price_timestamp(); // Use as "last activity" timestamp + self.health.update_price_timestamp(); - // Initialize aggregated table if needed - info!("๐Ÿ“‹ Step 1/7: Initializing aggregated data table..."); - self.init_aggregated_table()?; - - // Tier 1: Aggregate raw data older than 24 hours into 1-minute buckets info!("๐Ÿ“Š Step 2/7: Aggregating raw data into 1-minute buckets..."); - let aggregated_1m = self.aggregate_data(60, RAW_DATA_RETENTION_HOURS * 3600)?; + let aggregated_1m = self + .aggregate_data(60, RAW_DATA_RETENTION_HOURS * 3600) + .await?; - // Tier 2: Aggregate 1-minute data older than 7 days into 5-minute buckets info!("๐Ÿ“Š Step 3/7: Aggregating 1-minute data into 5-minute buckets..."); - // CHANGED: Source from 60s buckets instead of raw data - let aggregated_5m = - self.aggregate_buckets(60, 300, MINUTE_DATA_RETENTION_DAYS * 24 * 3600)?; + let aggregated_5m = self + .aggregate_buckets(60, 300, MINUTE_DATA_RETENTION_DAYS * 24 * 3600) + .await?; - // Tier 3: Aggregate 5-minute data older than 30 days into 15-minute buckets info!("๐Ÿ“Š Step 4/7: Aggregating 5-minute data into 15-minute buckets..."); - // CHANGED: Source from 300s buckets instead of raw data - let aggregated_15m = - self.aggregate_buckets(300, 900, FIVE_MINUTE_DATA_RETENTION_DAYS * 24 * 3600)?; + let aggregated_15m = self + .aggregate_buckets(300, 900, FIVE_MINUTE_DATA_RETENTION_DAYS * 24 * 3600) + .await?; - // Clean up raw data that has been aggregated (older than 24 hours) info!("๐Ÿ—‘๏ธ Step 5/7: Cleaning up old raw data (older than 24 hours)..."); - let deleted_raw = self.cleanup_aggregated_raw_data(RAW_DATA_RETENTION_HOURS * 3600)?; + let deleted_raw = self + .cleanup_aggregated_raw_data(RAW_DATA_RETENTION_HOURS * 3600) + .await?; - // Clean up old aggregated data beyond retention periods info!("๐Ÿ—‘๏ธ Step 6/7: Cleaning up old aggregated data..."); - let deleted_1m = self.cleanup_old_aggregates(60, MINUTE_DATA_RETENTION_DAYS * 24 * 3600)?; - let deleted_5m = - self.cleanup_old_aggregates(300, FIVE_MINUTE_DATA_RETENTION_DAYS * 24 * 3600)?; - let deleted_15m = - self.cleanup_old_aggregates(900, FIFTEEN_MINUTE_DATA_RETENTION_DAYS * 24 * 3600)?; + let deleted_1m = self + .cleanup_old_aggregates(60, MINUTE_DATA_RETENTION_DAYS * 24 * 3600) + .await?; + let deleted_5m = self + .cleanup_old_aggregates(300, FIVE_MINUTE_DATA_RETENTION_DAYS * 24 * 3600) + .await?; + let deleted_15m = self + .cleanup_old_aggregates(900, FIFTEEN_MINUTE_DATA_RETENTION_DAYS * 24 * 3600) + .await?; - // Vacuum database if significant cleanup occurred let total_deleted = deleted_raw + deleted_1m + deleted_5m + deleted_15m; if total_deleted > 1000 { info!( - "๐Ÿ”ง Step 7/7: Running database vacuum (deleted {} records)...", + "๐Ÿ”ง Step 7/7: Skipping vacuum for PostgreSQL (deleted {} records)", total_deleted ); - self.vacuum_database()?; } else { info!( "โญ๏ธ Step 7/7: Skipping vacuum (only {} records deleted)", @@ -702,12 +572,10 @@ impl DatabaseCleanup { ); } - // Update health timestamp self.health.update_db_timestamp(); - // Show final statistics info!("๐Ÿ“ˆ Generating final database statistics..."); - self.get_database_stats()?; + self.get_database_stats().await?; info!("โœ… Cleanup cycle completed:"); info!( @@ -719,7 +587,6 @@ impl DatabaseCleanup { Ok(()) } - /// Run the cleanup service with periodic execution pub async fn run(&self) -> BotResult<()> { let interval_hours = std::env::var("CLEANUP_INTERVAL_HOURS") .unwrap_or_else(|_| "24".to_string()) @@ -731,9 +598,6 @@ impl DatabaseCleanup { info!("๐Ÿš€ Database cleanup service started"); info!("โฐ Cleanup interval: {} hours", interval_hours); - // Note: Health server is now started by main.rs with aggregated health from all bots - - // Run initial cleanup after a short delay sleep(Duration::from_secs(30)).await; loop { diff --git a/src/errors.rs b/src/errors.rs index dd84296..6518712 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -3,7 +3,7 @@ use thiserror::Error; #[derive(Debug, Error)] pub enum BotError { #[error("Database error: {0}")] - Database(#[from] rusqlite::Error), + Database(String), #[error("HTTP request error: {0}")] Http(String), diff --git a/src/main.rs b/src/main.rs index 384acba..f9d4f4a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -41,7 +41,7 @@ async fn main() -> BotResult<()> { // Initialize shared database info!("๐Ÿ“ฆ Initializing shared database..."); - let db = match PriceDatabase::new(config::DATABASE_PATH) { + let db = match PriceDatabase::new(&config::DATABASE_URL).await { Ok(db) => Arc::new(db), Err(e) => { error!("Failed to initialize database: {}", e); diff --git a/src/price_service.rs b/src/price_service.rs index 44fd17c..be65366 100644 --- a/src/price_service.rs +++ b/src/price_service.rs @@ -539,7 +539,7 @@ pub async fn run( // Store in SQLite database using shared pool for (crypto, price_data) in &prices.prices { - if let Err(e) = database.save_price(crypto, price_data.price) { + if let Err(e) = database.save_price(crypto, price_data.price).await { error!("Failed to store {} price in database: {}", crypto, e); } } diff --git a/src/shanghai_price_service.rs b/src/shanghai_price_service.rs index c9425b3..b081e49 100644 --- a/src/shanghai_price_service.rs +++ b/src/shanghai_price_service.rs @@ -71,7 +71,7 @@ pub async fn run( price_data.premium_percent.unwrap_or(0.0) ); - if let Err(e) = database.save_price(CRYPTO_NAME, price_data.price) { + if let Err(e) = database.save_price(CRYPTO_NAME, price_data.price).await { error!("โŒ Failed to save Shanghai Silver price to database: {}", e); } else { info!("๐Ÿ’พ Saved Shanghai Silver price to database"); From 8e20d88119f59a2fbc2384714a787ff8fdc1c3a0 Mon Sep 17 00:00:00 2001 From: buzzkillb Date: Sun, 19 Apr 2026 14:14:53 -0700 Subject: [PATCH 8/9] feat: improve stability with task supervision and shared price state Issue 4 - DB Write Failures: - Add db_failures counter to HealthState - Increment on DB write failure, reset on success - Mark unhealthy if db_failures > 3 Issue 5 - Health Server Bind Failure: - Add start_health_server_with_retry() with 10 retries - Health server bind failure is now non-fatal - Bots continue running even if health server fails Issues 1 & 6 - Silent Task Death and Supervision: - All spawned tasks now have supervision loops - Services auto-restart on unexpected exit - Store and monitor all service join handles - Log CRITICAL errors when services die unexpectedly Issue 2 - Race Condition on prices.json: - Add SharedPrices with tokio::sync::RwLock - Price service writes to shared state atomically first - Bots read from shared state instead of file - File write happens AFTER atomic state update --- src/bot.rs | 114 ++++++--------------- src/health.rs | 16 +++ src/health_server.rs | 41 ++++++++ src/main.rs | 181 +++++++++++++++++++++++++--------- src/price_service.rs | 8 +- src/price_state.rs | 59 +++++++++++ src/shanghai_price_service.rs | 10 ++ 7 files changed, 296 insertions(+), 133 deletions(-) create mode 100644 src/price_state.rs diff --git a/src/bot.rs b/src/bot.rs index 9d8e04e..0c242b6 100644 --- a/src/bot.rs +++ b/src/bot.rs @@ -3,6 +3,7 @@ use crate::database::PriceDatabase; use crate::discord_api::DiscordApi; use crate::errors::{BotError, BotResult}; use crate::health::{HealthAggregator, HealthState}; +use crate::price_state::SharedPrices; use crate::charting::generate_price_chart; use crate::price_service::PricesFile; @@ -20,7 +21,6 @@ use serenity::{ Client, }; use std::collections::HashMap; -use std::fs; use std::sync::Arc; use std::time::Duration; use tokio::time::sleep; @@ -49,6 +49,7 @@ pub struct Bot { health: Arc, health_aggregator: Arc, database: Arc, + shared_prices: SharedPrices, } impl Bot { @@ -58,12 +59,14 @@ impl Bot { database: Arc, health: Arc, health_aggregator: Arc, + shared_prices: SharedPrices, ) -> BotResult { Ok(Self { config, health, health_aggregator, database, + shared_prices, }) } @@ -379,12 +382,13 @@ pub async fn start_bot( database: Arc, health: Arc, health_aggregator: Arc, + shared_prices: SharedPrices, ) -> BotResult<()> { let token = config.discord_token.clone(); let intents = GatewayIntents::GUILDS | GatewayIntents::GUILD_MESSAGES | GatewayIntents::MESSAGE_CONTENT; - let bot = Bot::new(config, database, health, health_aggregator)?; + let bot = Bot::new(config, database, health, health_aggregator, shared_prices)?; let mut client = Client::builder(&token, intents) .event_handler(bot) @@ -441,9 +445,10 @@ impl EventHandler for Bot { let config = self.config.clone(); let health = self.health.clone(); let database = self.database.clone(); + let shared_prices = self.shared_prices.clone(); tokio::spawn(async move { - price_update_loop(http, ctx_arc, config, health, database).await; + price_update_loop(http, ctx_arc, config, health, database, shared_prices).await; }); info!("Bot initialization complete!"); @@ -745,54 +750,6 @@ impl EventHandler for Bot { } } -/// Read prices from the shared JSON file with retry logic -async fn read_prices_from_file() -> BotResult { - let file_path = "shared/prices.json"; - const MAX_RETRIES: u32 = 3; - - for attempt in 1..=MAX_RETRIES { - // Check if file exists - if !std::path::Path::new(file_path).exists() { - if attempt < MAX_RETRIES { - warn!("Prices file not found (attempt {}), retrying...", attempt); - sleep(Duration::from_millis(1000 * attempt as u64)).await; - continue; - } - return Err(BotError::Io(std::io::Error::new( - std::io::ErrorKind::NotFound, - "Prices file not found. Make sure price-service is running.", - ))); - } - - match fs::read_to_string(file_path) { - Ok(content) => match serde_json::from_str::(&content) { - Ok(prices) => return Ok(prices), - Err(e) => { - error!("Failed to parse prices file (attempt {}): {}", attempt, e); - if attempt < MAX_RETRIES { - sleep(Duration::from_millis(1000 * attempt as u64)).await; - continue; - } - return Err(BotError::Json(e)); - } - }, - Err(e) => { - error!("Failed to read prices file (attempt {}): {}", attempt, e); - if attempt < MAX_RETRIES { - sleep(Duration::from_millis(1000 * attempt as u64)).await; - continue; - } - return Err(BotError::Io(e)); - } - } - } - - Err(BotError::Io(std::io::Error::new( - std::io::ErrorKind::Other, - "Unexpected error in prices file read retry loop", - ))) -} - /// Main price update loop with comprehensive error handling async fn price_update_loop( http: Arc, @@ -800,6 +757,7 @@ async fn price_update_loop( config: BotConfig, health: Arc, database: Arc, + shared_prices: SharedPrices, ) { let crypto_name = &config.crypto_name; let mut consecutive_failures = 0; @@ -813,7 +771,7 @@ async fn price_update_loop( loop { let loop_start = std::time::Instant::now(); - let current_price = match get_crypto_price(&config, &database).await { + let current_price = match get_crypto_price(&config, &database, &shared_prices).await { Ok(price) => { consecutive_failures = 0; health.reset_failures(); @@ -859,25 +817,15 @@ async fn price_update_loop( Err(_) => 0, }; - let custom_status = match read_prices_from_file().await { - Ok(shared_prices) => format_custom_status( - crypto_name, - current_price, - &shared_prices, - update_count, - &arrow, - change_percent, - ), - Err(e) => { - warn!("Failed to read shared prices for status: {}", e); - if change_percent == 0.0 && arrow == "๐Ÿ”„" { - format!("{} Building history", arrow) - } else { - let change_sign = if change_percent >= 0.0 { "+" } else { "" }; - format!("{} {}{:.2}% (1h)", arrow, change_sign, change_percent) - } - } - }; + let prices_data = shared_prices.read().await; + let custom_status = format_custom_status( + crypto_name, + current_price, + &prices_data, + update_count, + &arrow, + change_percent, + ); debug!("Updating nickname to: {}", nickname); debug!("Updating custom status to: {}", custom_status); @@ -887,8 +835,10 @@ async fn price_update_loop( if let Err(e) = database.save_price(crypto_name, current_price).await { error!("Failed to save price to database: {}", e); + health.increment_db_failures(); } else { health.update_db_timestamp(); + health.reset_db_failures(); } let guilds = ctx.cache.guilds(); @@ -1259,7 +1209,11 @@ fn format_custom_status( } /// Get current cryptocurrency price -async fn get_crypto_price(config: &BotConfig, database: &Arc) -> BotResult { +async fn get_crypto_price( + config: &BotConfig, + database: &Arc, + shared_prices: &SharedPrices, +) -> BotResult { // For SHANGHAISILVER, read directly from database (not in prices.json) if config.crypto_name == "SHANGHAISILVER" { debug!("Getting SHANGHAISILVER price from database"); @@ -1281,17 +1235,11 @@ async fn get_crypto_price(config: &BotConfig, database: &Arc) -> } } - // First try to get from shared prices file - match read_prices_from_file().await { - Ok(prices) => { - if let Some(price_data) = prices.prices.get(&config.crypto_name) { - validate_price(price_data.price)?; - return Ok(price_data.price); - } - } - Err(_) => { - // If shared file doesn't exist or doesn't have our crypto, try direct API call - } + // First try to get from shared prices state + let prices = shared_prices.read().await; + if let Some(price_data) = prices.prices.get(&config.crypto_name) { + validate_price(price_data.price)?; + return Ok(price_data.price); } // Fallback to direct API call if we have a feed ID diff --git a/src/health.rs b/src/health.rs index 1c13346..e7a4052 100644 --- a/src/health.rs +++ b/src/health.rs @@ -14,6 +14,7 @@ pub struct HealthState { pub consecutive_failures: Arc, pub gateway_failures: Arc, pub discord_test_failures: Arc, + pub db_failures: Arc, pub start_time: Arc, pub bot_name: String, } @@ -32,6 +33,7 @@ impl HealthState { consecutive_failures: Arc::new(AtomicU64::new(0)), gateway_failures: Arc::new(AtomicU64::new(0)), discord_test_failures: Arc::new(AtomicU64::new(0)), + db_failures: Arc::new(AtomicU64::new(0)), start_time: Arc::new(AtomicU64::new(start)), bot_name, } @@ -93,6 +95,14 @@ impl HealthState { self.discord_test_failures.store(0, Ordering::Relaxed); } + pub fn increment_db_failures(&self) { + self.db_failures.fetch_add(1, Ordering::Relaxed); + } + + pub fn reset_db_failures(&self) { + self.db_failures.store(0, Ordering::Relaxed); + } + pub fn get_uptime_seconds(&self) -> u64 { let now = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -116,6 +126,7 @@ impl HealthState { let failures = self.consecutive_failures.load(Ordering::Relaxed); let gateway_failures = self.gateway_failures.load(Ordering::Relaxed); let discord_test_failures = self.discord_test_failures.load(Ordering::Relaxed); + let db_failures = self.db_failures.load(Ordering::Relaxed); // Consider unhealthy if: // - No price update in last 5 minutes @@ -125,6 +136,7 @@ impl HealthState { // - More than 3 consecutive failures // - More than 5 gateway failures (indicates broken Discord connection) // - More than 3 Discord test failures (indicates connection issues) + // - More than 3 DB write failures (indicates DB issues) // Treat 0 (never updated) as using start_time for staleness check let price_stale = last_price > 0 && now.saturating_sub(last_price) > 300; let db_stale = last_db > 0 && now.saturating_sub(last_db) > 300; @@ -134,6 +146,7 @@ impl HealthState { let too_many_failures = failures > 3; let gateway_broken = gateway_failures > 5; let discord_test_broken = discord_test_failures > 3; + let db_broken = db_failures > 3; !price_stale && !db_stale @@ -142,6 +155,7 @@ impl HealthState { && !too_many_failures && !gateway_broken && !discord_test_broken + && !db_broken } pub fn to_json(&self) -> serde_json::Value { @@ -157,6 +171,7 @@ impl HealthState { let failures = self.consecutive_failures.load(Ordering::Relaxed); let gateway_failures = self.gateway_failures.load(Ordering::Relaxed); let discord_test_failures = self.discord_test_failures.load(Ordering::Relaxed); + let db_failures = self.db_failures.load(Ordering::Relaxed); let uptime = self.get_uptime_seconds(); json!({ @@ -171,6 +186,7 @@ impl HealthState { "consecutive_failures": failures, "gateway_failures": gateway_failures, "discord_test_failures": discord_test_failures, + "db_failures": db_failures, "seconds_since_price_update": now.saturating_sub(last_price), "seconds_since_db_write": now.saturating_sub(last_db), "seconds_since_discord_update": now.saturating_sub(last_discord), diff --git a/src/health_server.rs b/src/health_server.rs index e650100..0fe9ff4 100644 --- a/src/health_server.rs +++ b/src/health_server.rs @@ -39,6 +39,47 @@ pub async fn start_health_server( Ok(()) } +pub async fn start_health_server_with_retry( + health: SharedHealth, + port: u16, + max_retries: u32, +) -> Result<(), Box> { + let addr = format!("127.0.0.1:{}", port); + + for attempt in 1..=max_retries { + match TcpListener::bind(&addr).await { + Ok(listener) => { + info!( + "Health check server listening on {} (attempt {})", + addr, attempt + ); + let app = Router::new() + .route("/health", get(health_check)) + .route("/health/all", get(health_check_all)) + .route("/", get(health_check)) + .route("/test-discord", get(test_discord_connectivity)) + .with_state(health); + return Ok(axum::serve(listener, app).await.map_err(|e| e.into())); + } + Err(e) => { + error!( + "Failed to bind health server to {} (attempt {}/{}): {}", + addr, attempt, max_retries, e + ); + if attempt < max_retries { + tokio::time::sleep(Duration::from_secs(5)).await; + } + } + } + } + + Err(format!( + "Failed to bind health server after {} attempts", + max_retries + ) + .into()) +} + async fn health_check(State(health): State) -> Response { let health = health.clone(); let result = timeout( diff --git a/src/main.rs b/src/main.rs index f9d4f4a..577af8b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,6 +10,7 @@ mod errors; mod health; mod health_server; mod price_service; +mod price_state; mod shanghai_price_service; mod utils; @@ -19,7 +20,8 @@ use database::PriceDatabase; use db_cleanup::DatabaseCleanup; use errors::BotResult; use health::{HealthAggregator, HealthState}; -use health_server::start_health_server; +use health_server::start_health_server_with_retry; +use price_state::SharedPrices; use dotenv::dotenv; use std::sync::Arc; @@ -28,12 +30,27 @@ use tokio::time::sleep; use tracing::{error, info, warn}; const RECONNECT_DELAY_SECONDS: u64 = 30; +const SERVICE_RESTART_DELAY_SECONDS: u64 = 5; + +enum ServiceHandle { + Bot(tokio::task::JoinHandle<()>, String), + Service(tokio::task::JoinHandle<()>, String), + HealthServer(tokio::task::JoinHandle<()>), +} + +fn format_service_name(handle: &ServiceHandle) -> &str { + match handle { + ServiceHandle::Bot(_, name) => name.as_str(), + ServiceHandle::Service(_, name) => name.as_str(), + ServiceHandle::HealthServer(_) => "health_server", + } +} #[tokio::main] async fn main() -> BotResult<()> { // Initialize logging tracing_subscriber::fmt() - .with_env_filter("info,discord_bot=debug,discord_bot::database=info") + .with_env_filter("info,rustymcpriceface=debug") .init(); info!("๐Ÿš€ Starting RustyMcPriceface Unified Container..."); @@ -49,39 +66,74 @@ async fn main() -> BotResult<()> { } }; - // Start Database Cleanup Service + // Create shared price state for all services and bots + let shared_prices = Arc::new(SharedPrices::new()); + + // Create health aggregator for all bots + let health_aggregator = Arc::new(HealthAggregator::new()); + + // Storage for all service handles + let mut service_handles: Vec = Vec::new(); + + // Spawn Database Cleanup Service with supervision info!("๐Ÿงน Starting Database Cleanup Service..."); - { - let db_clone = db.clone(); - tokio::spawn(async move { - let cleanup = DatabaseCleanup::new(db_clone); - if let Err(e) = cleanup.run().await { - error!("Cleanup service crashed: {}", e); + let db_cleanup_db = db.clone(); + let cleanup_handle = tokio::spawn(async move { + let cleanup = DatabaseCleanup::new(db_cleanup_db); + loop { + info!("๐Ÿงน Cleanup service starting..."); + match cleanup.run().await { + Ok(_) => { + error!("๐Ÿงน Cleanup service exited unexpectedly - restarting in {}s", SERVICE_RESTART_DELAY_SECONDS); + } + Err(e) => { + error!("๐Ÿงน Cleanup service crashed: {} - restarting in {}s", e, SERVICE_RESTART_DELAY_SECONDS); + } } - }); - } + sleep(Duration::from_secs(SERVICE_RESTART_DELAY_SECONDS)).await; + } + }); + service_handles.push(ServiceHandle::Service(cleanup_handle, "cleanup".to_string())); - // Start Price Service + // Spawn Price Service with supervision info!("๐Ÿ’น Starting Price Fetching Service..."); - { - let db_clone = db.clone(); - tokio::spawn(async move { - if let Err(e) = price_service::run(db_clone).await { - error!("Price service crashed: {}", e); + let price_service_db = db.clone(); + let price_service_prices = shared_prices.clone(); + let price_handle = tokio::spawn(async move { + loop { + info!("๐Ÿ’น Price service starting..."); + match price_service::run(price_service_db.clone(), price_service_prices.clone()).await { + Ok(_) => { + error!("๐Ÿ’น Price service exited unexpectedly - restarting in {}s", SERVICE_RESTART_DELAY_SECONDS); + } + Err(e) => { + error!("๐Ÿ’น Price service crashed: {} - restarting in {}s", e, SERVICE_RESTART_DELAY_SECONDS); + } } - }); - } + sleep(Duration::from_secs(SERVICE_RESTART_DELAY_SECONDS)).await; + } + }); + service_handles.push(ServiceHandle::Service(price_handle, "price_service".to_string())); - // Start Shanghai Silver Price Service + // Spawn Shanghai Silver Price Service with supervision info!("๐Ÿญ Starting Shanghai Silver Price Service..."); - { - let db_clone = db.clone(); - tokio::spawn(async move { - if let Err(e) = shanghai_price_service::run(db_clone).await { - error!("Shanghai price service crashed: {}", e); + let shanghai_db = db.clone(); + let shanghai_prices = shared_prices.clone(); + let shanghai_handle = tokio::spawn(async move { + loop { + info!("๐Ÿญ Shanghai price service starting..."); + match shanghai_price_service::run(shanghai_db.clone(), shanghai_prices.clone()).await { + Ok(_) => { + error!("๐Ÿญ Shanghai price service exited unexpectedly - restarting in {}s", SERVICE_RESTART_DELAY_SECONDS); + } + Err(e) => { + error!("๐Ÿญ Shanghai price service crashed: {} - restarting in {}s", e, SERVICE_RESTART_DELAY_SECONDS); + } } - }); - } + sleep(Duration::from_secs(SERVICE_RESTART_DELAY_SECONDS)).await; + } + }); + service_handles.push(ServiceHandle::Service(shanghai_handle, "shanghai_price_service".to_string())); // Load all bot instances let instances = BotConfig::load_bot_instances(); @@ -94,15 +146,11 @@ async fn main() -> BotResult<()> { // Global configuration for update interval let global_config = BotConfig::from_env()?; - // Create health aggregator for all bots - let health_aggregator = Arc::new(HealthAggregator::new()); - // Spawn a task for each bot - let mut handles = vec![]; - for (ticker, token) in instances { let db_clone = db.clone(); let health_agg_clone = health_aggregator.clone(); + let bot_prices = shared_prices.clone(); let mut bot_config = global_config.clone(); bot_config.crypto_name = ticker.clone(); bot_config.discord_token = token.clone(); @@ -116,9 +164,8 @@ async fn main() -> BotResult<()> { info!("๐Ÿš€ Spawning bot for {}...", ticker); - let handle = tokio::spawn(async move { + let bot_handle = tokio::spawn(async move { loop { - // Determine appropriate emoji for logs let emoji = utils::get_crypto_emoji(&ticker); info!("{} Starting {} bot...", emoji, ticker); @@ -127,6 +174,7 @@ async fn main() -> BotResult<()> { db_clone.clone(), health_clone.clone(), health_agg_clone.clone(), + bot_prices.clone(), ) .await { @@ -145,32 +193,69 @@ async fn main() -> BotResult<()> { sleep(Duration::from_secs(RECONNECT_DELAY_SECONDS)).await; } }); - handles.push(handle); + service_handles.push(ServiceHandle::Bot(bot_handle, ticker)); } - // Start health check server + // Start health check server (non-fatal - retries but doesn't crash container) info!("๐Ÿฅ Starting health check server..."); let health_for_server = health_aggregator.clone(); - tokio::spawn(async move { - if let Err(e) = start_health_server(health_for_server, 8080).await { - error!("โŒ Health server failed: {}", e); - panic!("Health server must start successfully for container health checks"); + let health_handle = tokio::spawn(async move { + match start_health_server_with_retry(health_for_server, 8080, 10).await { + Ok(_) => { + error!("๐Ÿฅ Health server exited unexpectedly"); + } + Err(e) => { + error!("๐Ÿฅ Health server failed after retries: {}", e); + } + } + // Don't restart health server - if it can't bind, something is wrong + // The bots should continue running regardless + loop { + sleep(Duration::from_secs(60)).await; } }); + service_handles.push(ServiceHandle::HealthServer(health_handle)); // Give health server time to start sleep(Duration::from_secs(1)).await; - // Keep the main process alive - if !handles.is_empty() { - info!("โœ… All bots spawned. Main process entering monitor loop."); - // Wait for all handles (they shouldn't return unless panicked/cancelled) - for handle in handles { - let _ = handle.await; + // Monitor all service handles + if !service_handles.is_empty() { + info!("โœ… All services spawned. Monitoring {} services...", service_handles.len()); + + loop { + // Check all handles + let mut all_dead = true; + let mut dead_services = Vec::new(); + + for handle in &service_handles { + let is_dead = match handle { + ServiceHandle::Bot(h, name) => h.is_finished(), + ServiceHandle::Service(h, name) => h.is_finished(), + ServiceHandle::HealthServer(h) => h.is_finished(), + }; + + if !is_dead { + all_dead = false; + } else { + dead_services.push(handle); + } + } + + // If any service died, log fatal error (they should restart themselves) + if !dead_services.is_empty() { + for handle in &dead_services { + let name = format_service_name(handle); + error!("๐Ÿ’€ CRITICAL: {} died unexpectedly - it should auto-restart", name); + } + } + + // Sleep before next check + sleep(Duration::from_secs(5)).await; } } else { - warn!("โš ๏ธ No bots to run. Exiting."); + warn!("โš ๏ธ No services to run. Exiting."); } Ok(()) -} +} \ No newline at end of file diff --git a/src/price_service.rs b/src/price_service.rs index be65366..7b5c27f 100644 --- a/src/price_service.rs +++ b/src/price_service.rs @@ -492,6 +492,7 @@ fn extract_first_number_after(html: &str, prefix: &str) -> Option { pub async fn run( database: Arc, + shared_prices: Arc, ) -> Result<(), Box> { // Get update interval from environment let update_interval = std::env::var("UPDATE_INTERVAL_SECONDS") @@ -532,12 +533,15 @@ pub async fn run( Ok(prices) => { consecutive_failures = 0; // Reset failure counter on success - // Store in JSON file (for backward compatibility) + // Write to shared state FIRST (atomic update) + shared_prices.write(prices.clone()).await; + + // THEN persist to file (for backward compatibility / debugging) if let Err(e) = write_prices_to_file(&prices, &file_path).await { error!("Failed to write prices to JSON: {}", e); } - // Store in SQLite database using shared pool + // Store in database using shared pool for (crypto, price_data) in &prices.prices { if let Err(e) = database.save_price(crypto, price_data.price).await { error!("Failed to store {} price in database: {}", crypto, e); diff --git a/src/price_state.rs b/src/price_state.rs new file mode 100644 index 0000000..1a1d153 --- /dev/null +++ b/src/price_state.rs @@ -0,0 +1,59 @@ +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::sync::Arc; +use tokio::sync::RwLock; + +#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] +pub struct PriceData { + pub price: f64, + pub timestamp: u64, + #[serde(default)] + pub premium: Option, + #[serde(default)] + pub premium_percent: Option, + pub source: Option, + #[serde(default)] + pub is_fallback: bool, +} + +#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] +pub struct PricesFile { + pub prices: HashMap, + pub timestamp: u64, +} + +#[derive(Debug, Clone)] +pub struct SharedPrices { + inner: Arc>, +} + +impl SharedPrices { + pub fn new() -> Self { + Self { + inner: Arc::new(RwLock::new(PricesFile { + prices: HashMap::new(), + timestamp: 0, + })), + } + } + + pub async fn write(&self, prices: PricesFile) { + let mut current = self.inner.write().await; + *current = prices; + } + + pub async fn read(&self) -> PricesFile { + self.inner.read().await.clone() + } + + pub async fn get_price(&self, crypto: &str) -> Option { + let current = self.inner.read().await; + current.prices.get(crypto).map(|p| p.price) + } +} + +impl Default for SharedPrices { + fn default() -> Self { + Self::new() + } +} diff --git a/src/shanghai_price_service.rs b/src/shanghai_price_service.rs index b081e49..3dc48f4 100644 --- a/src/shanghai_price_service.rs +++ b/src/shanghai_price_service.rs @@ -1,5 +1,6 @@ use crate::database::PriceDatabase; use crate::price_service::{fetch_shanghai_silver_price, PriceData, PricesFile}; +use crate::price_state::SharedPrices; use chrono::Utc; use std::collections::HashMap; use std::fs; @@ -41,6 +42,7 @@ fn is_sge_market_open() -> bool { pub async fn run( database: Arc, + shared_prices: Arc, ) -> Result<(), Box> { let update_interval = get_update_interval(); info!("๐Ÿš€ Starting Shanghai Silver Price Service..."); @@ -77,6 +79,14 @@ pub async fn run( info!("๐Ÿ’พ Saved Shanghai Silver price to database"); } + // Update shared prices state first + let mut shared_prices_data = shared_prices.read().await; + shared_prices_data + .prices + .insert(CRYPTO_NAME.to_string(), price_data.clone()); + shared_prices.write(shared_prices_data).await; + info!("๐Ÿ“ Updated shared prices state"); + if let Err(e) = update_prices_json(&price_data) { error!("โŒ Failed to update prices.json: {}", e); } else { From bd70916c9fcd57bd209045bf1ce96228de69c678 Mon Sep 17 00:00:00 2001 From: buzzkillb Date: Sun, 19 Apr 2026 14:39:26 -0700 Subject: [PATCH 9/9] fix: compilation errors in stability migration - Fix syntax error in database.rs (extra parenthesis) - Fix Arc/SharedPrices type mismatches across modules - Fix i64 vs u64 type casts in db_cleanup.rs - Fix duplicate Arc import in price_service.rs - Fix private struct re-exports from price_state - Fix ticker move before use in async spawn loop - Fix health_server return type in match arm --- src/bot.rs | 10 ++--- src/database.rs | 40 +++++++++++++++----- src/db_cleanup.rs | 18 +++++---- src/health_server.rs | 2 +- src/main.rs | 70 ++++++++++++++++++++++++++--------- src/price_service.rs | 21 +---------- src/shanghai_price_service.rs | 4 +- 7 files changed, 103 insertions(+), 62 deletions(-) diff --git a/src/bot.rs b/src/bot.rs index 0c242b6..5b7100a 100644 --- a/src/bot.rs +++ b/src/bot.rs @@ -6,7 +6,7 @@ use crate::health::{HealthAggregator, HealthState}; use crate::price_state::SharedPrices; use crate::charting::generate_price_chart; -use crate::price_service::PricesFile; +use crate::price_state::PricesFile; use crate::utils::{format_price, get_current_timestamp, validate_crypto_name, validate_price}; use serenity::{ all::{ @@ -49,7 +49,7 @@ pub struct Bot { health: Arc, health_aggregator: Arc, database: Arc, - shared_prices: SharedPrices, + shared_prices: Arc, } impl Bot { @@ -59,7 +59,7 @@ impl Bot { database: Arc, health: Arc, health_aggregator: Arc, - shared_prices: SharedPrices, + shared_prices: Arc, ) -> BotResult { Ok(Self { config, @@ -382,7 +382,7 @@ pub async fn start_bot( database: Arc, health: Arc, health_aggregator: Arc, - shared_prices: SharedPrices, + shared_prices: Arc, ) -> BotResult<()> { let token = config.discord_token.clone(); let intents = @@ -757,7 +757,7 @@ async fn price_update_loop( config: BotConfig, health: Arc, database: Arc, - shared_prices: SharedPrices, + shared_prices: Arc, ) { let crypto_name = &config.crypto_name; let mut consecutive_failures = 0; diff --git a/src/database.rs b/src/database.rs index 4ed9ff9..bbd39c5 100644 --- a/src/database.rs +++ b/src/database.rs @@ -161,13 +161,17 @@ impl PriceDatabase { }; let old_price = if seconds <= 24 * 3600 { - self.get_price_from_raw_data(crypto, time_ago as i64).await? + self.get_price_from_raw_data(crypto, time_ago as i64) + .await? } else if seconds <= 7 * 24 * 3600 { - self.get_price_from_aggregates(crypto, time_ago as i64, 60).await? + self.get_price_from_aggregates(crypto, time_ago as i64, 60) + .await? } else if seconds < 30 * 24 * 3600 { - self.get_price_from_aggregates(crypto, time_ago as i64, 300).await? + self.get_price_from_aggregates(crypto, time_ago as i64, 300) + .await? } else { - self.get_price_from_aggregates(crypto, time_ago as i64, 900).await? + self.get_price_from_aggregates(crypto, time_ago as i64, 900) + .await? }; if let Some(price) = old_price { @@ -181,7 +185,7 @@ impl PriceDatabase { changes.push(format!( "{} {}{:.2}% ({})", arrow, sign, change_percent, label - ))); + )); } else { debug!( "No {} {} price data found for time_ago: {}", @@ -217,7 +221,12 @@ impl PriceDatabase { Ok(row.map(|r| r.0)) } - async fn get_price_from_aggregates(&self, crypto: &str, time_ago: i64, bucket_duration: i64) -> BotResult> { + async fn get_price_from_aggregates( + &self, + crypto: &str, + time_ago: i64, + bucket_duration: i64, + ) -> BotResult> { let row: Option<(f64,)> = sqlx::query_as( "SELECT open_price FROM price_aggregates WHERE crypto_name = $1 AND bucket_duration = $2 @@ -234,7 +243,11 @@ impl PriceDatabase { Ok(row.map(|r| r.0)) } - pub async fn get_price_indicator(&self, crypto_name: &str, current_price: f64) -> (String, f64) { + pub async fn get_price_indicator( + &self, + crypto_name: &str, + current_price: f64, + ) -> (String, f64) { let current_time = match get_current_timestamp() { Ok(time) => time as i64, Err(_) => return ("๐Ÿ”„".to_string(), 0.0), @@ -267,7 +280,11 @@ impl PriceDatabase { ("๐Ÿ”„".to_string(), 0.0) } - pub async fn get_price_history(&self, crypto_name: &str, days: u64) -> BotResult> { + pub async fn get_price_history( + &self, + crypto_name: &str, + days: u64, + ) -> BotResult> { let current_time = get_current_timestamp()? as i64; let start_time = current_time - (days as i64 * 86400); @@ -347,7 +364,10 @@ impl PriceDatabase { .map_err(|e| BotError::Database(e.to_string()))?; if result.rows_affected() > 0 { - info!("Cleaned up {} old price records from database", result.rows_affected()); + info!( + "Cleaned up {} old price records from database", + result.rows_affected() + ); } Ok(()) @@ -366,4 +386,4 @@ impl PriceDatabase { } } } -} \ No newline at end of file +} diff --git a/src/db_cleanup.rs b/src/db_cleanup.rs index d18b623..ffdfa23 100644 --- a/src/db_cleanup.rs +++ b/src/db_cleanup.rs @@ -530,33 +530,37 @@ impl DatabaseCleanup { info!("๐Ÿ“Š Step 2/7: Aggregating raw data into 1-minute buckets..."); let aggregated_1m = self - .aggregate_data(60, RAW_DATA_RETENTION_HOURS * 3600) + .aggregate_data(60, (RAW_DATA_RETENTION_HOURS * 3600) as i64) .await?; info!("๐Ÿ“Š Step 3/7: Aggregating 1-minute data into 5-minute buckets..."); let aggregated_5m = self - .aggregate_buckets(60, 300, MINUTE_DATA_RETENTION_DAYS * 24 * 3600) + .aggregate_buckets(60, 300, (MINUTE_DATA_RETENTION_DAYS * 24 * 3600) as i64) .await?; info!("๐Ÿ“Š Step 4/7: Aggregating 5-minute data into 15-minute buckets..."); let aggregated_15m = self - .aggregate_buckets(300, 900, FIVE_MINUTE_DATA_RETENTION_DAYS * 24 * 3600) + .aggregate_buckets( + 300, + 900, + (FIVE_MINUTE_DATA_RETENTION_DAYS * 24 * 3600) as i64, + ) .await?; info!("๐Ÿ—‘๏ธ Step 5/7: Cleaning up old raw data (older than 24 hours)..."); let deleted_raw = self - .cleanup_aggregated_raw_data(RAW_DATA_RETENTION_HOURS * 3600) + .cleanup_aggregated_raw_data((RAW_DATA_RETENTION_HOURS * 3600) as i64) .await?; info!("๐Ÿ—‘๏ธ Step 6/7: Cleaning up old aggregated data..."); let deleted_1m = self - .cleanup_old_aggregates(60, MINUTE_DATA_RETENTION_DAYS * 24 * 3600) + .cleanup_old_aggregates(60, (MINUTE_DATA_RETENTION_DAYS * 24 * 3600) as i64) .await?; let deleted_5m = self - .cleanup_old_aggregates(300, FIVE_MINUTE_DATA_RETENTION_DAYS * 24 * 3600) + .cleanup_old_aggregates(300, (FIVE_MINUTE_DATA_RETENTION_DAYS * 24 * 3600) as i64) .await?; let deleted_15m = self - .cleanup_old_aggregates(900, FIFTEEN_MINUTE_DATA_RETENTION_DAYS * 24 * 3600) + .cleanup_old_aggregates(900, (FIFTEEN_MINUTE_DATA_RETENTION_DAYS * 24 * 3600) as i64) .await?; let total_deleted = deleted_raw + deleted_1m + deleted_5m + deleted_15m; diff --git a/src/health_server.rs b/src/health_server.rs index 0fe9ff4..ae99f92 100644 --- a/src/health_server.rs +++ b/src/health_server.rs @@ -59,7 +59,7 @@ pub async fn start_health_server_with_retry( .route("/", get(health_check)) .route("/test-discord", get(test_discord_connectivity)) .with_state(health); - return Ok(axum::serve(listener, app).await.map_err(|e| e.into())); + return axum::serve(listener, app).await.map_err(|e| e.into()); } Err(e) => { error!( diff --git a/src/main.rs b/src/main.rs index 577af8b..4084da6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -79,21 +79,30 @@ async fn main() -> BotResult<()> { info!("๐Ÿงน Starting Database Cleanup Service..."); let db_cleanup_db = db.clone(); let cleanup_handle = tokio::spawn(async move { - let cleanup = DatabaseCleanup::new(db_cleanup_db); + let cleanup = DatabaseCleanup::new(&db_cleanup_db); loop { info!("๐Ÿงน Cleanup service starting..."); match cleanup.run().await { Ok(_) => { - error!("๐Ÿงน Cleanup service exited unexpectedly - restarting in {}s", SERVICE_RESTART_DELAY_SECONDS); + error!( + "๐Ÿงน Cleanup service exited unexpectedly - restarting in {}s", + SERVICE_RESTART_DELAY_SECONDS + ); } Err(e) => { - error!("๐Ÿงน Cleanup service crashed: {} - restarting in {}s", e, SERVICE_RESTART_DELAY_SECONDS); + error!( + "๐Ÿงน Cleanup service crashed: {} - restarting in {}s", + e, SERVICE_RESTART_DELAY_SECONDS + ); } } sleep(Duration::from_secs(SERVICE_RESTART_DELAY_SECONDS)).await; } }); - service_handles.push(ServiceHandle::Service(cleanup_handle, "cleanup".to_string())); + service_handles.push(ServiceHandle::Service( + cleanup_handle, + "cleanup".to_string(), + )); // Spawn Price Service with supervision info!("๐Ÿ’น Starting Price Fetching Service..."); @@ -104,16 +113,25 @@ async fn main() -> BotResult<()> { info!("๐Ÿ’น Price service starting..."); match price_service::run(price_service_db.clone(), price_service_prices.clone()).await { Ok(_) => { - error!("๐Ÿ’น Price service exited unexpectedly - restarting in {}s", SERVICE_RESTART_DELAY_SECONDS); + error!( + "๐Ÿ’น Price service exited unexpectedly - restarting in {}s", + SERVICE_RESTART_DELAY_SECONDS + ); } Err(e) => { - error!("๐Ÿ’น Price service crashed: {} - restarting in {}s", e, SERVICE_RESTART_DELAY_SECONDS); + error!( + "๐Ÿ’น Price service crashed: {} - restarting in {}s", + e, SERVICE_RESTART_DELAY_SECONDS + ); } } sleep(Duration::from_secs(SERVICE_RESTART_DELAY_SECONDS)).await; } }); - service_handles.push(ServiceHandle::Service(price_handle, "price_service".to_string())); + service_handles.push(ServiceHandle::Service( + price_handle, + "price_service".to_string(), + )); // Spawn Shanghai Silver Price Service with supervision info!("๐Ÿญ Starting Shanghai Silver Price Service..."); @@ -124,16 +142,25 @@ async fn main() -> BotResult<()> { info!("๐Ÿญ Shanghai price service starting..."); match shanghai_price_service::run(shanghai_db.clone(), shanghai_prices.clone()).await { Ok(_) => { - error!("๐Ÿญ Shanghai price service exited unexpectedly - restarting in {}s", SERVICE_RESTART_DELAY_SECONDS); + error!( + "๐Ÿญ Shanghai price service exited unexpectedly - restarting in {}s", + SERVICE_RESTART_DELAY_SECONDS + ); } Err(e) => { - error!("๐Ÿญ Shanghai price service crashed: {} - restarting in {}s", e, SERVICE_RESTART_DELAY_SECONDS); + error!( + "๐Ÿญ Shanghai price service crashed: {} - restarting in {}s", + e, SERVICE_RESTART_DELAY_SECONDS + ); } } sleep(Duration::from_secs(SERVICE_RESTART_DELAY_SECONDS)).await; } }); - service_handles.push(ServiceHandle::Service(shanghai_handle, "shanghai_price_service".to_string())); + service_handles.push(ServiceHandle::Service( + shanghai_handle, + "shanghai_price_service".to_string(), + )); // Load all bot instances let instances = BotConfig::load_bot_instances(); @@ -164,10 +191,11 @@ async fn main() -> BotResult<()> { info!("๐Ÿš€ Spawning bot for {}...", ticker); + let ticker_for_handle = ticker.clone(); let bot_handle = tokio::spawn(async move { loop { - let emoji = utils::get_crypto_emoji(&ticker); - info!("{} Starting {} bot...", emoji, ticker); + let emoji = utils::get_crypto_emoji(&ticker_for_handle); + info!("{} Starting {} bot...", emoji, ticker_for_handle); match start_bot( bot_config.clone(), @@ -179,16 +207,16 @@ async fn main() -> BotResult<()> { .await { Ok(_) => { - error!("{} {} bot exited unexpectedly", emoji, ticker); + error!("{} {} bot exited unexpectedly", emoji, ticker_for_handle); } Err(e) => { - error!("{} {} bot crashed: {}", emoji, ticker, e); + error!("{} {} bot crashed: {}", emoji, ticker_for_handle, e); } } error!( "{} Restarting {} bot in {} seconds...", - emoji, ticker, RECONNECT_DELAY_SECONDS + emoji, ticker_for_handle, RECONNECT_DELAY_SECONDS ); sleep(Duration::from_secs(RECONNECT_DELAY_SECONDS)).await; } @@ -221,7 +249,10 @@ async fn main() -> BotResult<()> { // Monitor all service handles if !service_handles.is_empty() { - info!("โœ… All services spawned. Monitoring {} services...", service_handles.len()); + info!( + "โœ… All services spawned. Monitoring {} services...", + service_handles.len() + ); loop { // Check all handles @@ -246,7 +277,10 @@ async fn main() -> BotResult<()> { if !dead_services.is_empty() { for handle in &dead_services { let name = format_service_name(handle); - error!("๐Ÿ’€ CRITICAL: {} died unexpectedly - it should auto-restart", name); + error!( + "๐Ÿ’€ CRITICAL: {} died unexpectedly - it should auto-restart", + name + ); } } @@ -258,4 +292,4 @@ async fn main() -> BotResult<()> { } Ok(()) -} \ No newline at end of file +} diff --git a/src/price_service.rs b/src/price_service.rs index 7b5c27f..1fdf608 100644 --- a/src/price_service.rs +++ b/src/price_service.rs @@ -1,26 +1,16 @@ +use crate::price_state::{PriceData, PricesFile, SharedPrices}; use reqwest; use serde_json::Value; use std::collections::HashMap; use std::fs; use std::path::Path; +use std::sync::Arc; use std::time::Duration; use tokio::time::sleep; use tracing::{error, info, warn}; const HERMES_API_URL: &str = "https://hermes.pyth.network/api/latest_price_feeds"; -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] -pub struct PriceData { - pub price: f64, - pub timestamp: u64, - // Optional fields for detailed data (e.g., Shanghai Premium) - pub premium: Option, - pub premium_percent: Option, - pub source: Option, - #[serde(default)] - pub is_fallback: bool, -} - #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] pub struct HistoryData { pub date: String, // "YYYY-MM-DD" @@ -31,12 +21,6 @@ pub struct HistoryData { pub premium_percent: f64, } -#[derive(serde::Serialize, serde::Deserialize)] -pub struct PricesFile { - pub prices: HashMap, - pub timestamp: u64, -} - fn get_feed_ids() -> HashMap { let mut feeds = HashMap::new(); @@ -386,7 +370,6 @@ async fn write_prices_to_file( } use crate::database::PriceDatabase; -use std::sync::Arc; const GOLDSILVER_AI_URL: &str = "https://goldsilver.ai/metal-prices/shanghai-silver-price"; diff --git a/src/shanghai_price_service.rs b/src/shanghai_price_service.rs index 3dc48f4..99d3abe 100644 --- a/src/shanghai_price_service.rs +++ b/src/shanghai_price_service.rs @@ -1,6 +1,6 @@ use crate::database::PriceDatabase; -use crate::price_service::{fetch_shanghai_silver_price, PriceData, PricesFile}; -use crate::price_state::SharedPrices; +use crate::price_service::fetch_shanghai_silver_price; +use crate::price_state::{PriceData, PricesFile, SharedPrices}; use chrono::Utc; use std::collections::HashMap; use std::fs;