Implement Retry Logic with Exponential Backoff

Cargo.toml

@@ -7,7 +7,6 @@ members = [
     "examples/token",
     "examples/escrow",
     "examples/vesting",
-    "backend",
 ]
 resolver = "2"
 

backend/Cargo.toml

@@ -2,163 +2,89 @@
 name = "backend"
 version = "0.1.0"
 edition = "2021"
-
-[dependencies]
-axum = "0.7"
-tokio = { version = "1", features = ["full"] }
-serde = { version = "1", features = ["derive"] }
-serde_json = "1"
-sqlx = { version = "0.7", features = ["runtime-tokio-rustls", "postgres", "chrono", "uuid"] }
-redis = { version = "0.25", features = ["tokio-comp"] }
-tracing = "0.1"
-tracing-subscriber = { version = "0.3", features = ["env-filter"] }
-thiserror = "1.0"
-chrono = { version = "0.4", features = ["serde"] }
-uuid = { version = "1", features = ["v4", "serde"] }
-dotenvy = "0.15"
-tower-http = { version = "0.5", features = ["trace"] }
-name = "crucible-backend"
-version = "0.1.0"
-edition = "2021"
 description = "Backend API server for the Crucible smart contract testing platform"
 license = "MIT"
 authors = ["Crucible Contributors"]
+publish = false
 
 [[bin]]
 name = "crucible-backend"
 path = "src/main.rs"
 
+[[bin]]
+name = "backup"
+path = "src/bin/backup.rs"
+
+[features]
+default = []
+testutils = ["mockall"]
+
 [dependencies]
-# Web framework
+# Web framework & HTTP
 axum = { version = "0.7", features = ["macros"] }
-tower = { version = "0.4", features = ["full"] }
+tower = { version = "0.5", features = ["util"] }
 tower-http = { version = "0.5", features = ["cors", "trace", "compression-gzip", "request-id"] }
 
 # Async runtime
 tokio = { version = "1", features = ["full"] }
 
-# Database
-sqlx = { version = "0.7", features = [
-    "runtime-tokio-rustls",
-    "postgres",
-    "uuid",
-    "chrono",
-    "json",
-    "migrate",
-] }
+# Database (PostgreSQL via SQLx)
+sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "postgres", "chrono", "uuid", "json", "macros", "migrate", "rust_decimal"] }
 
 # Redis
-redis = { version = "0.25", features = ["tokio-comp", "connection-manager"] }
+redis = { version = "0.27", features = ["tokio-comp", "connection-manager", "json"] }
 
 # Serialization
-serde = { version = "1", features = ["derive"] }
-serde_json = "1"
-
-# Observability
-tracing = "0.1"
-tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
-
-# Utilities
-uuid = { version = "1", features = ["v4", "serde"] }
-chrono = { version = "0.4", features = ["serde"] }
-dotenvy = "0.15"
-thiserror = "1"
-
-[dev-dependencies]
-# Testing
-reqwest = { version = "0.12", features = ["json"] }
-tokio-test = "0.4"
-testcontainers = "0.16"
-wiremock = "0.6"
-
-[profile.release]
-opt-level = 3
-lto = true
-codegen-units = 1
-strip = true
-
-[dependencies]
-axum = "0.7"
-sqlx = { version = "0.7", features = ["postgres", "runtime-tokio", "macros"] }
-redis = { version = "0.25", features = ["tokio-comp"] }
-tokio = { version = "1.0", features = ["full"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 schemars = "0.8"
-tracing = "0.1"
-tracing-subscriber = "0.3"
-
-[dev-dependencies]
-tower = "0.4"
-name = "backend"
-version = "0.1.0"
-edition = "2021"
-
-[[bin]]
-name = "backup"
-path = "src/bin/backup.rs"
-[features]
-testutils = ["mockall"]
 
-[dependencies]
-axum = "0.7"
-tokio = { version = "1", features = ["full"] }
-serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
-sqlx = { version = "0.7", features = ["runtime-tokio-rustls", "postgres", "macros", "chrono", "uuid"] }
-redis = { version = "0.24", features = ["tokio-comp", "json"] }
-sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "postgres", "chrono", "uuid", "json"] }
-redis = { version = "0.27", features = ["tokio-comp", "json"] }
-tracing = "0.1"
-tracing-subscriber = { version = "0.3", features = ["env-filter"] }
-anyhow = "1.0"
-thiserror = "1.0"
-chrono = { version = "0.4", features = ["serde"] }
+# Utilities
 uuid = { version = "1.0", features = ["v4", "serde"] }
-tower = { version = "0.5", features = ["util"] }
-tower-http = { version = "0.5", features = ["trace"] }
-
-[dev-dependencies]
-tower = { version = "0.5", features = ["util"] }
-hyper = { version = "1.0", features = ["full"] }
-mime = "0.3"
-tokio = { version = "1", features = ["full", "test-util"] }
-arc-swap = "1.7"
-async-trait = "0.1"
+chrono = { version = "0.4", features = ["serde"] }
 dotenvy = "0.15"
-utoipa = { version = "5.0", features = ["axum_extras", "chrono", "uuid"] }
-utoipa-swagger-ui = { version = "8.0", features = ["axum"] }
-apalis = { version = "0.6" }
-apalis-redis = "0.6"
-rust_decimal = { version = "1.35", features = ["serde"] }
+thiserror = "1.0"
+anyhow = "1.0"
+rust_decimal = { version = "1.35", features = ["serde", "db-postgres"] }
+rust_decimal_macros = "1.35"
+arc-swap = "1.7"
 stellar-xdr = { version = "21.0", features = ["std"] }
 base64 = "0.22"
 validator = { version = "0.19", features = ["derive"] }
-tower-http = { version = "0.5", features = ["cors", "trace"] }
-tower_governor = "0.4"
-mockall = { version = "0.13", optional = true }
-opentelemetry = { version = "0.31", features = ["trace"] }
-opentelemetry_sdk = { version = "0.31", features = ["trace", "rt-tokio"] }
-opentelemetry-otlp = { version = "0.31", default-features = false, features = ["trace", "http-proto", "reqwest-client"] }
-tracing-opentelemetry = { version = "0.32", default-features = false }
 futures-util = { version = "0.3", default-features = false, features = ["std"] }
-# OpenTelemetry and tracing instrumentation
+
+# Background jobs
+apalis = { version = "0.6" }
+apalis-redis = "0.6"
+
+# Observability / Telemetry
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
 opentelemetry = { version = "0.24", features = ["trace", "metrics"] }
+opentelemetry_sdk = { version = "0.24", features = ["trace", "rt-tokio"] }
 opentelemetry-otlp = { version = "0.17", features = ["trace", "grpc-tonic"] }
 opentelemetry-semantic-conventions = "0.16"
-opentelemetry_sdk = { version = "0.24", features = ["trace", "rt-tokio"] }
 tracing-opentelemetry = "0.25"
 tonic = "0.12"
 
+# Optional dependencies
+mockall = { version = "0.13", optional = true }
+
+# API Docs
+utoipa = { version = "5.0", features = ["axum_extras", "chrono", "uuid", "decimal"] }
+utoipa-swagger-ui = { version = "8.0", features = ["axum"] }
+tower_governor = "0.4"
+
 [dev-dependencies]
-tower = { version = "0.4", features = ["util"] }
-tower-http = { version = "0.5", features = ["trace"] }
-rust_decimal_macros = "1.35"
+reqwest = { version = "0.12", features = ["json"] }
+tokio-test = "0.4"
+testcontainers = "0.16"
+wiremock = "0.6"
 criterion = { version = "0.5", features = ["async_tokio"] }
-hyper = { version = "1.0", features = ["full"] }
 mime = "0.3"
+hyper = { version = "1.0", features = ["full"] }
+async-trait = "0.1"
 mockall = "0.13"
-mockall = "0.12"
 
 [[bench]]
 name = "performance"
@@ -167,4 +93,3 @@ harness = false
 [[bench]]
 name = "dashboard_bench"
 harness = false
-

backend/README.md

@@ -1141,3 +1141,32 @@ impl Validate for ProfileTriggerRequest {
 - `src/jobs/` – Background job definitions (Apalis)
 - `src/services/` – Business logic and external integrations
 - `src/telemetry/` – Observability and logging setup
+- `src/workers/` – Background worker modules, including the exponential backoff retry policy
+
+## Exponential Backoff Retry Policy
+
+Crucible includes a production-ready retry module (`src/workers/retry.rs`) to safely retry fallible asynchronous operations (such as SQLx database transactions, Redis calls, or external Stellar Horizon requests).
+
+### Features
+- **Exponential Backoff**: Dynamically increases backoff duration using the formula `base_delay * multiplier^attempt`.
+- **Full Jitter**: Implements the AWS recommended full jitter strategy to prevent thundering herd problems.
+- **Conditional Retries**: Abort retries early based on the specific error encountered using the `ShouldRetry` predicate.
+- **Observability**: Built-in structured tracing (`tracing` spans and logs) to track retry attempts, backoff durations, and exhausted errors.
+- **Serializable Configuration**: `RetryConfig` supports direct Serialization/Deserialization for config-driven retries.
+
+### Usage Example
+```rust
+use backend::workers::retry::{RetryPolicy, RetryError};
+
+async fn perform_stellar_tx() -> Result<(), MyError> {
+    let policy = RetryPolicy::default()
+        .with_base_delay(std::time::Duration::from_millis(100))
+        .with_max_delay(std::time::Duration::from_secs(10));
+
+    policy.retry(|| async {
+        // Fallible operation
+        send_transaction().await
+    }).await.map_err(|err| err.into_inner())
+}
+```
+