From 5c266e508cceaaa26f78c8c18ff66ca5823554f3 Mon Sep 17 00:00:00 2001 From: zizou <111426680+zizou0x@users.noreply.github.com> Date: Mon, 20 Jul 2026 17:24:01 +0200 Subject: [PATCH] feat(config): add remote config layer fetched from S3 Per-chain tuned values are pulled at startup and applied between the embedded defaults and the local config file. Fetching is fail-safe: bounded retries, size cap, explicit redirect handling, and any error falls back to the lower layers with a warning. Lib users get config::get_default(chain) for embedded + remote in one call. Co-Authored-By: Claude Fable 5 --- .claude/CODEBASE.md | 3 +- docs/guides/server-configuration.md | 6 +- fynd-core/CLAUDE.md | 2 +- fynd-core/src/config/mod.rs | 26 +- fynd-core/src/config/remote.rs | 416 ++++++++++++++++++++++++++++ src/cli.rs | 29 +- src/main.rs | 59 +++- 7 files changed, 518 insertions(+), 23 deletions(-) create mode 100644 fynd-core/src/config/remote.rs diff --git a/.claude/CODEBASE.md b/.claude/CODEBASE.md index 4b4f8712c..4f7d451d5 100644 --- a/.claude/CODEBASE.md +++ b/.claude/CODEBASE.md @@ -103,6 +103,7 @@ See `docs/ARCHITECTURE.md` for the full architecture diagram and detailed compon | `HTTP_HOST` | HTTP bind address (default: `0.0.0.0`) | | `HTTP_PORT` | API port (default: `3000`) | | `CONFIG_FILE` | Solver config file (default: `fynd.toml` if present) | +| `REMOTE_CONFIG_URL` | Remote config URL (default: chain-specific PropellerHeads S3 URL; `--no-remote-config` disables) | | `WORKER_POOLS_CONFIG` | Deprecated legacy pools-only config file; its pools override the config file's | | `BLOCKLIST_CONFIG` | Blocklist config file | | `RUST_LOG` | Tracing filter (e.g. `info,fynd=debug`) | @@ -120,7 +121,7 @@ See `docs/ARCHITECTURE.md` for the full architecture diagram and detailed compon | File | Purpose | |---|---| -| `fynd.toml` | Full solver config: any subset of tuning fields + `[pools]`. Resolved field-by-field: CLI > file > embedded default (`fynd-core/src/config/default_config.toml`) | +| `fynd.toml` | Full solver config: any subset of tuning fields + `[pools]`. Resolved field-by-field: CLI > file > remote config (S3, per chain) > embedded default (`fynd-core/src/config/default_config.toml`) | | `worker_pools.toml` | Deprecated legacy pools-only file, still honored (its pools override the config file's) | | `blocklist.toml` | Component IDs to exclude from the Tycho stream. Optional — falls back to tycho-simulation defaults if not found | diff --git a/docs/guides/server-configuration.md b/docs/guides/server-configuration.md index 475ecc8ec..80473db06 100644 --- a/docs/guides/server-configuration.md +++ b/docs/guides/server-configuration.md @@ -87,6 +87,8 @@ Run `fynd serve --help` for the full list. | `--worker-router-timeout-ms` | — | `100` | Default solve timeout (ms) | | `--worker-router-min-responses` | — | `0` | Early return threshold (0 = wait for all pools) | | `--config-file` | `CONFIG_FILE` | `fynd.toml` (if present) | TOML config file overriding the embedded defaults (see [Config file](#config-file-fyndtoml)). | +| `--remote-config-url` | `REMOTE_CONFIG_URL` | _(chain-specific S3 URL)_ | Remote config with the latest tuned values, pulled at startup. Fetch failures never block startup. | +| `--no-remote-config` | — | `false` | Disable the remote config fetch. | | `-w, --worker-pools-config` | `WORKER_POOLS_CONFIG` | `worker_pools.toml` (if present) | **Deprecated** — legacy pools-only config file; move the `[pools]` section into `fynd.toml`. Still honored: its pools override the config file's. | | `--blocklist-config` | `BLOCKLIST_CONFIG` | [tycho-simulation default](https://github.com/propeller-heads/tycho-simulation/blob/main/blocklist.toml) | Path to blocklist TOML config file. Components listed here are excluded from the Tycho stream. | | `--disable-tls` | — | `false` | Disable TLS for Tycho connection | @@ -104,8 +106,8 @@ Run `fynd serve --help` for the full list. ## Config file (`fynd.toml`) -Every solver-tuning flag above resolves field-by-field through three layers, highest priority -first: **CLI flags > config file > embedded defaults**. The config file may set any subset of +Every solver-tuning flag above resolves field-by-field through four layers, highest priority +first: **CLI flags > config file > remote config (S3) > embedded defaults**. The config file may set any subset of the fields (same names as the flags) plus the worker pools; `./fynd.toml` is picked up automatically, or pass `--config-file`. diff --git a/fynd-core/CLAUDE.md b/fynd-core/CLAUDE.md index 28c0177bb..01fafe23e 100644 --- a/fynd-core/CLAUDE.md +++ b/fynd-core/CLAUDE.md @@ -9,7 +9,7 @@ applications. |-----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `algorithm/` | `Algorithm` trait + built-in `MostLiquidAlgorithm`, `BellmanFordAlgorithm`, `PathFrankWolfeAlgorithm`. Pluggable via associated graph types. `AlgorithmConfig` shared by all | | `solver.rs` | `FyndBuilder` assembles the full pipeline (feed + gas + computations + pools + encoder + router). `Solver` runs it | -| `config/` | Layered solver config: `embedded_default()` parses `default_config.toml` (single source of truth for all tuning defaults) into a complete `Config`; `PartialConfig` layers overlay via `Config::apply` (CLI/file > embedded); `Config::validate` range-checks the result | +| `config/` | Layered solver config: `embedded_default()` parses `default_config.toml` (single source of truth for all tuning defaults) into a complete `Config`; `PartialConfig` layers overlay via `Config::apply` (CLI > file > remote > embedded); `config::remote` fetches per-chain tuned values from S3 (fail-safe: retries, size cap, warn-and-fallback, never panics), `get_default(chain)` = embedded + remote in one call; `Config::validate` range-checks the result | | `worker_pool/` | `WorkerPool` manages dedicated OS threads. `SolverWorker` runs a prioritized select loop (shutdown > market events > derived events > tasks). `TaskQueue` is `async_channel`-based | | `worker_pool_router/` | `WorkerPoolRouter` fans out orders to all pools, ranks candidates by `amount_out_net_gas` descending; price guard (if enabled) validates in rank order; optionally encodes | | `feed/` | `TychoFeed` (WebSocket → MarketState), `GasPriceFetcher`, `MarketEvent` broadcasting, `ProtocolRegistry` | diff --git a/fynd-core/src/config/mod.rs b/fynd-core/src/config/mod.rs index 2ec8a1b9a..01f8e59e9 100644 --- a/fynd-core/src/config/mod.rs +++ b/fynd-core/src/config/mod.rs @@ -5,7 +5,8 @@ //! //! 1. **Explicit overrides** — lib builder setters or CLI flags //! 2. **Local config file** — any subset of the fields, same schema as the embedded default -//! 3. **Embedded default** — `default_config.toml`, compiled into the binary +//! 3. **Remote config** — tuned values pulled from S3 per chain (see [`remote`]) +//! 4. **Embedded default** — `default_config.toml`, compiled into the binary //! //! The embedded default deserializes directly into a complete [`Config`] — every field //! (except the chain-specific `min_tvl`) is required, so a gap between the struct and @@ -26,6 +27,8 @@ //! let overrides = PartialConfig { worker_router_timeout_ms: Some(50), ..Default::default() }; //! let config = embedded_default() //! .clone() +//! .apply_remote(&remote::default_remote_config_url(chain), timeout) +//! .await //! .apply(&PartialConfig::from_file("fynd.toml")?) //! .apply(&overrides); //! let builder = FyndBuilder::new(chain, tycho_url, rpc_url, config.protocols.clone(), min_tvl) @@ -39,6 +42,8 @@ use tycho_simulation::tycho_common::models::{Chain, TvlThresholdTier}; use crate::solver::PoolConfig; +pub mod remote; + /// The embedded default configuration, compiled into the binary. const EMBEDDED_DEFAULT_TOML: &str = include_str!("default_config.toml"); @@ -151,6 +156,25 @@ pub fn embedded_default() -> &'static Config { &EMBEDDED_DEFAULT } +/// Overall time budget (including retries) for the fetch inside [`get_default`]. +/// Callers wanting a different budget use [`Config::apply_remote`] directly. +const GET_DEFAULT_FETCH_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2); + +/// Returns the embedded default configuration with the latest remotely tuned values for +/// `chain` applied on top, fetched from the default S3 URL (see +/// [`remote::default_remote_config_url`]). +/// +/// The simple one-call form of `embedded_default().clone().apply_remote(...)`, with a +/// built-in 2 s fetch budget. Never fails or panics: on any fetch problem the embedded +/// defaults are returned unchanged (a warning is logged). Layer local overrides on top +/// with [`Config::apply`]; for a custom URL or timeout use [`Config::apply_remote`]. +pub async fn get_default(chain: Chain) -> Config { + embedded_default() + .clone() + .apply_remote(&remote::default_remote_config_url(chain), GET_DEFAULT_FETCH_TIMEOUT) + .await +} + impl Config { /// Returns `min_tvl`, falling back to `chain`'s default TVL threshold when unset. pub fn min_tvl_or_chain_default(&self, chain: Chain) -> f64 { diff --git a/fynd-core/src/config/remote.rs b/fynd-core/src/config/remote.rs new file mode 100644 index 000000000..458c0907b --- /dev/null +++ b/fynd-core/src/config/remote.rs @@ -0,0 +1,416 @@ +//! Remote config layer: tuned values pulled from S3 at startup. +//! +//! # Layering +//! +//! The remote payload is a [`PartialConfig`] in the same schema as the local config file, +//! published per chain. It applies **right above the embedded default**: +//! +//! ```text +//! CLI flags > local config file (fynd.toml) > remote config (S3) > embedded default +//! ``` +//! +//! Operators' local settings therefore always beat remotely tuned values. +//! +//! # Safety properties +//! +//! The remote layer must never take a solver down or degrade it silently: +//! +//! 1. **Never blocks or fails startup.** The caller bounds the whole fetch (including retries) with +//! [`tokio::time::timeout`]; on any error or timeout it logs a warning and resolves without the +//! remote layer — the embedded default covers everything. +//! 2. **Bounded, transient-only retries.** Connection-level failures and 5xx responses are retried +//! up to a few times with linear backoff; 4xx responses and parse failures are deterministic and +//! fail immediately. +//! 3. **Size-capped download.** The response body is size-capped before parsing, so a misconfigured +//! URL cannot exhaust memory. +//! 4. **No panic paths.** Every failure — client construction, transport, decoding, parsing, +//! payload checks — returns an error; nothing in this module unwraps, expects, or panics. The +//! caller downgrades every error to a warning and resolves without the remote layer. +//! 5. **Forward-compatible parsing, backward-safe application.** Unknown fields in the payload are +//! ignored (a payload written for a newer binary still applies the fields this binary knows), +//! but a payload whose pools reference an algorithm this binary does not implement is rejected +//! wholesale — an outdated binary falls back to its embedded defaults instead of failing at +//! solver build time. +//! 6. **No validation bypass.** The remote layer is folded like any other; the final resolved +//! config still passes [`Config::validate`](super::Config::validate) (and +//! `FyndBuilder::apply_config` validates again at the engine boundary). A remote payload that +//! would produce an invalid config fails resolution the same way a bad local file does — +//! visibly, at startup. +//! +//! # Deliberate non-features (open for review) +//! +//! - **No on-disk cache / TTL.** The fetch happens once at startup and the embedded default covers +//! the offline case, so a cache only adds staleness questions. +//! - **No schema version field.** Forward compatibility comes from permissive parsing; an +//! incompatible future schema would fail parsing and fall back safely. +//! - **No payload signing.** Integrity currently rests on TLS + bucket ACLs. If the bucket becomes +//! a broader attack surface, a detached signature (e.g. sidecar `latest.toml.sig` verified +//! against an embedded public key) can be added without changing this API. + +use std::time::Duration; + +use tycho_simulation::tycho_common::models::Chain; + +use super::{Config, ConfigError, PartialConfig}; +use crate::worker_pool::registry::AVAILABLE_ALGORITHMS; + +/// URL template behind [`default_remote_config_url`]; `{chain}` is substituted with the +/// lowercase chain name. +const DEFAULT_REMOTE_URL_TEMPLATE: &str = + "https://s3.eu-central-1.amazonaws.com/repo.propellerheads-propellerheads/fynd/presets/{chain}/latest.toml"; + +/// Number of attempts for transient network failures before giving up. +const FETCH_ATTEMPTS: u32 = 3; + +/// Base delay between retries; grows linearly with the attempt number. +const RETRY_BACKOFF: Duration = Duration::from_millis(150); + +/// Maximum accepted response body size. A config payload is a few KiB; anything near this +/// limit is misconfiguration or abuse. +const MAX_RESPONSE_BYTES: u64 = 256 * 1024; + +/// Returns the default remote config URL for `chain` (the PropellerHeads-maintained S3 +/// object with the latest tuned values). +pub fn default_remote_config_url(chain: Chain) -> String { + DEFAULT_REMOTE_URL_TEMPLATE.replace("{chain}", &chain.to_string()) +} + +/// Errors from fetching the remote config. +/// +/// All variants are recoverable: callers log a warning and resolve without the remote +/// layer. +#[derive(Debug, thiserror::Error)] +pub enum RemoteConfigError { + /// Network-level failure: DNS, connect, or non-2xx status (after retries). + #[error("request failed: {0}")] + Request(#[from] reqwest::Error), + /// The response body exceeded the size limit. + #[error("response exceeds the {limit_bytes} byte limit")] + TooLarge { + /// The enforced limit. + limit_bytes: u64, + }, + /// The server answered with a non-success status outside plain 4xx/5xx — typically a + /// redirect reqwest cannot follow (S3 omits the Location header when the URL targets + /// the wrong region/endpoint for the bucket). + #[error( + "unexpected status {status} (redirect? check that the URL matches the bucket's region)" + )] + UnexpectedStatus { + /// The status the server answered with. + status: reqwest::StatusCode, + }, + /// The response body is not valid UTF-8. + #[error("response body is not valid UTF-8")] + InvalidUtf8, + /// The payload failed to parse as a [`PartialConfig`]. + #[error(transparent)] + Parse(#[from] ConfigError), + /// A pool in the payload references an algorithm this binary does not know — typically + /// a payload written for a newer version. Falling back to the embedded defaults keeps + /// outdated binaries running instead of failing startup at solver build time. + #[error( + "remote config pool '{pool}' uses unknown algorithm '{algorithm}' \ + (this binary supports: {available})" + )] + UnknownAlgorithm { + /// The pool naming the unknown algorithm. + pool: String, + /// The unknown algorithm name. + algorithm: String, + /// Comma-separated algorithm names this binary supports. + available: String, + }, +} + +impl Config { + /// Fetches the remote config layer from `url` and applies it on top of this config. + /// + /// The remote layer for the `apply` chain: + /// + /// ```ignore + /// let config = embedded_default() + /// .clone() + /// .apply_remote(&default_remote_config_url(chain), timeout) + /// .await + /// .apply(&local_file) + /// .apply(&overrides); + /// ``` + /// + /// For embedded + remote with the default URL in one call, use + /// [`get_default`](super::get_default). + /// + /// `timeout` bounds the whole fetch including retries. This method can never fail or + /// panic (safety properties 1 and 4): on any fetch error or timeout it logs a warning + /// and returns `self` unchanged. + pub async fn apply_remote(self, url: &str, timeout: Duration) -> Self { + match tokio::time::timeout(timeout, fetch_remote_config(url)).await { + Ok(Ok(partial)) => { + tracing::info!(url, "fetched remote config"); + self.apply(&partial) + } + Ok(Err(e)) => { + tracing::warn!(url, error = %e, "remote config fetch failed; continuing without it"); + self + } + Err(_elapsed) => { + tracing::warn!( + url, + timeout_ms = timeout.as_millis() as u64, + "remote config fetch timed out; continuing without it" + ); + self + } + } + } +} + +/// Fetches the remote config from `url` and returns it as a [`PartialConfig`] layer. +/// +/// Prefer [`Config::apply_remote`] unless you need the raw layer or custom error handling. +/// +/// Transient network failures are retried a few times with a short backoff. There is no +/// internal deadline: bound the call with [`tokio::time::timeout`] — the timeout is caller +/// policy. Build the default `url` with [`default_remote_config_url`]. +/// +/// # Errors +/// +/// Returns [`RemoteConfigError`] on any transport, size, parse, or payload-safety problem. +/// Treat every error as non-fatal: warn and resolve without the remote layer. +pub async fn fetch_remote_config(url: &str) -> Result { + let mut attempt = 1; + let body = loop { + match http_get_capped(url).await { + Ok(body) => break body, + Err(e) if attempt < FETCH_ATTEMPTS && is_retryable(&e) => { + tracing::debug!(url, attempt, error = %e, "remote config fetch failed; retrying"); + tokio::time::sleep(RETRY_BACKOFF * attempt).await; + attempt += 1; + } + Err(e) => return Err(e), + } + }; + + let partial = PartialConfig::from_toml_str(&body, url)?; + check_payload(&partial)?; + Ok(partial) +} + +/// One GET attempt with the response body capped at [`MAX_RESPONSE_BYTES`] +/// (safety property 3). +async fn http_get_capped(url: &str) -> Result { + // The `Client::new()` shortcut panics if the TLS backend fails to initialize; the + // builder reports it as an error instead (safety property 4). + let client = reqwest::Client::builder().build()?; + // `error_for_status` only covers 4xx/5xx; catch everything else non-2xx (e.g. an + // unfollowable 301 from S3, which omits the Location header on region mismatches) + // explicitly instead of handing an error document to the parser. + let mut response = client + .get(url) + .send() + .await? + .error_for_status()?; + if !response.status().is_success() { + return Err(RemoteConfigError::UnexpectedStatus { status: response.status() }); + } + + if let Some(length) = response.content_length() { + if length > MAX_RESPONSE_BYTES { + return Err(RemoteConfigError::TooLarge { limit_bytes: MAX_RESPONSE_BYTES }); + } + } + // Content-Length can be absent or lie; enforce the cap on the actual stream too. + let mut body = Vec::new(); + while let Some(chunk) = response.chunk().await? { + if (body.len() + chunk.len()) as u64 > MAX_RESPONSE_BYTES { + return Err(RemoteConfigError::TooLarge { limit_bytes: MAX_RESPONSE_BYTES }); + } + body.extend_from_slice(&chunk); + } + String::from_utf8(body).map_err(|_| RemoteConfigError::InvalidUtf8) +} + +/// True for failures worth retrying: connection-level errors and 5xx responses +/// (safety property 2). 4xx responses and oversized bodies are deterministic and fail +/// immediately. +fn is_retryable(error: &RemoteConfigError) -> bool { + match error { + RemoteConfigError::Request(e) => match e.status() { + Some(status) => status.is_server_error(), + None => true, + }, + RemoteConfigError::TooLarge { .. } | + RemoteConfigError::UnexpectedStatus { .. } | + RemoteConfigError::InvalidUtf8 | + RemoteConfigError::Parse(_) | + RemoteConfigError::UnknownAlgorithm { .. } => false, + } +} + +/// Rejects payloads whose pools reference algorithms this binary does not implement, so +/// they fail here (recoverable, falls back to embedded) rather than at solver build time +/// (fatal) — safety property 5. +fn check_payload(partial: &PartialConfig) -> Result<(), RemoteConfigError> { + let Some(pools) = &partial.pools else { + return Ok(()); + }; + for (pool_name, pool) in pools { + if !AVAILABLE_ALGORITHMS.contains(&pool.algorithm()) { + return Err(RemoteConfigError::UnknownAlgorithm { + pool: pool_name.clone(), + algorithm: pool.algorithm().to_string(), + available: AVAILABLE_ALGORITHMS.join(", "), + }); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Minimal HTTP/1.1 server answering one canned response per expected request, in + /// order. Returns the URL to fetch. + async fn spawn_server(responses: Vec<(u16, Vec)>) -> String { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind failed"); + let addr = listener + .local_addr() + .expect("no local addr"); + tokio::spawn(async move { + for (status, body) in responses { + let (mut socket, _) = listener + .accept() + .await + .expect("accept failed"); + let mut request = [0u8; 1024]; + let _ = socket.read(&mut request).await; + let head = format!( + "HTTP/1.1 {status} TEST\r\ncontent-length: {}\r\nconnection: close\r\n\r\n", + body.len() + ); + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.write_all(&body).await; + } + }); + format!("http://{addr}/latest.toml") + } + + #[tokio::test] + async fn test_fetch_applies_valid_payload() { + let url = spawn_server(vec![(200, b"worker_router_timeout_ms = 123".to_vec())]).await; + let partial = fetch_remote_config(&url) + .await + .expect("fetch errored"); + assert_eq!(partial.worker_router_timeout_ms, Some(123)); + assert_eq!(partial.min_token_quality, None); + } + + #[tokio::test] + async fn test_fetch_retries_transient_5xx() { + let url = spawn_server(vec![ + (500, b"".to_vec()), + (503, b"".to_vec()), + (200, b"min_token_quality = 90".to_vec()), + ]) + .await; + let partial = fetch_remote_config(&url) + .await + .expect("fetch errored"); + assert_eq!(partial.min_token_quality, Some(90)); + } + + #[tokio::test] + async fn test_fetch_does_not_retry_4xx() { + // A single 404 response; a retry would hang on the closed listener, so completing + // with an error proves no retry happened. + let url = spawn_server(vec![(404, b"".to_vec())]).await; + let error = fetch_remote_config(&url) + .await + .unwrap_err(); + assert!(matches!(error, RemoteConfigError::Request(_))); + } + + #[tokio::test] + async fn test_fetch_rejects_unfollowable_redirect() { + // S3 answers 301 without a Location header when the URL targets the wrong + // region; the XML error body must not reach the parser. + let url = spawn_server(vec![(301, b"".to_vec())]).await; + let error = fetch_remote_config(&url) + .await + .unwrap_err(); + assert!(matches!(error, RemoteConfigError::UnexpectedStatus { .. })); + assert!(error.to_string().contains("301")); + } + + #[tokio::test] + async fn test_fetch_rejects_oversized_body() { + let huge = vec![b'#'; (MAX_RESPONSE_BYTES + 1) as usize]; + let url = spawn_server(vec![(200, huge)]).await; + let error = fetch_remote_config(&url) + .await + .unwrap_err(); + assert!(matches!(error, RemoteConfigError::TooLarge { .. })); + } + + #[tokio::test] + async fn test_fetch_rejects_garbage_payload() { + let url = spawn_server(vec![(200, b"not [ valid { toml".to_vec())]).await; + let error = fetch_remote_config(&url) + .await + .unwrap_err(); + assert!(matches!(error, RemoteConfigError::Parse(_))); + } + + #[tokio::test] + async fn test_fetch_rejects_unknown_algorithm_payload() { + let url = + spawn_server(vec![(200, b"[pools.p]\nalgorithm = \"quantum_router\"".to_vec())]).await; + let error = fetch_remote_config(&url) + .await + .unwrap_err(); + assert!(matches!(error, RemoteConfigError::UnknownAlgorithm { .. })); + } + + #[tokio::test] + async fn test_apply_remote_falls_back_on_fetch_failure() { + // Port 9 (discard) refuses connections; the config must come through unchanged, + // never a panic or an error. + let base = super::super::embedded_default().clone(); + let config = base + .clone() + .apply_remote("http://127.0.0.1:9/latest.toml", Duration::from_millis(200)) + .await; + assert_eq!(config, base); + } + + #[test] + fn test_default_remote_config_url() { + assert_eq!( + default_remote_config_url(Chain::Ethereum), + "https://s3.eu-central-1.amazonaws.com/repo.propellerheads-propellerheads/fynd/presets/ethereum/latest.toml" + ); + assert_eq!( + default_remote_config_url(Chain::Base), + "https://s3.eu-central-1.amazonaws.com/repo.propellerheads-propellerheads/fynd/presets/base/latest.toml" + ); + } + + #[test] + fn test_check_payload_rejects_unknown_algorithm() { + let known = PartialConfig::from_toml_str("[pools.p]\nalgorithm = \"bellman_ford\"", "test") + .expect("parse errored"); + assert!(check_payload(&known).is_ok()); + + // A payload written for a newer binary must be recoverable, not fail startup. + let unknown = + PartialConfig::from_toml_str("[pools.p]\nalgorithm = \"quantum_router\"", "test") + .expect("parse errored"); + assert!(matches!(check_payload(&unknown), Err(RemoteConfigError::UnknownAlgorithm { .. }))); + + assert!(check_payload(&PartialConfig::default()).is_ok()); + } +} diff --git a/src/cli.rs b/src/cli.rs index 96afe8fe9..265464ec6 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -2,7 +2,11 @@ use std::path::PathBuf; use clap::Parser; use fynd_core::config::{embedded_default, PartialConfig}; -use fynd_rpc::config::{defaults, WorkerPoolsConfig}; +use fynd_rpc::{ + config::{defaults, WorkerPoolsConfig}, + parse_chain, +}; +use tycho_simulation::tycho_common::models::Chain; #[cfg(feature = "metrics")] pub(crate) const METRICS_PORT: u16 = 9898; @@ -39,14 +43,15 @@ pub enum Commands { /// Arguments for the `serve` subcommand. /// -/// Solver-tuning flags resolve field-by-field through three layers, highest priority first: +/// Solver-tuning flags resolve field-by-field through four layers, highest priority first: /// explicit CLI flags, the local config file (`--config-file`, default `fynd.toml` if -/// present), and the default config embedded in the binary. +/// present), the remote config pulled from S3, and the default config embedded in the +/// binary. #[derive(clap::Args, PartialEq, Debug)] pub struct ServeArgs { /// Target chain (e.g. Ethereum) - #[arg(short, long, default_value = "Ethereum")] - pub chain: String, + #[arg(short, long, default_value = "Ethereum", value_parser = parse_chain)] + pub chain: Chain, /// Path to a local TOML config file overriding the embedded defaults. Any subset of /// the config fields, same schema as the embedded default config. @@ -54,6 +59,16 @@ pub struct ServeArgs { #[arg(long, env)] pub config_file: Option, + /// URL of the remote config pulled at startup. Defaults to the chain-specific + /// PropellerHeads S3 URL. Fetch failures never block startup. + #[arg(long, env)] + pub remote_config_url: Option, + + /// Disable fetching the remote config; resolve from CLI, local file, and embedded + /// defaults only + #[arg(long)] + pub no_remote_config: bool, + /// HTTP host (e.g. 0.0.0.0) #[arg(long, default_value = defaults::HTTP_HOST, env)] pub http_host: String, @@ -236,7 +251,7 @@ mod cli_tests { let Commands::Serve(args) = cli.command else { panic!("expected Serve command"); }; - assert_eq!(args.chain, "Ethereum"); + assert_eq!(args.chain, Chain::Ethereum); assert_eq!(args.http_host, "127.0.0.1"); assert_eq!(args.http_port, 8080); assert_eq!(args.tycho_api_key, Some("test-key".to_string())); @@ -263,7 +278,7 @@ mod cli_tests { let Commands::Serve(args) = cli.command else { panic!("expected Serve command"); }; - assert_eq!(args.chain, "Ethereum"); + assert_eq!(args.chain, Chain::Ethereum); assert_eq!(args.http_host, "0.0.0.0"); assert_eq!(args.http_port, 3000); assert_eq!(args.tycho_api_key, None); diff --git a/src/main.rs b/src/main.rs index 83601be1f..62d6cff9f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -35,11 +35,10 @@ use std::{path::Path, time::Duration}; use actix_web::{web, App, HttpResponse, HttpServer, Responder}; use anyhow::anyhow; use clap::Parser; -use fynd_core::config::{embedded_default, PartialConfig}; +use fynd_core::config::{embedded_default, remote, PartialConfig}; use fynd_rpc::{ builder::FyndRPCBuilder, config::{defaults, BlocklistConfig, WorkerPoolsConfig}, - parse_chain, protocols::resolve_protocols, }; mod cli; @@ -226,11 +225,19 @@ const DEFAULT_CONFIG_PATH: &str = "fynd.toml"; /// local-file priority when no higher layer sets pools. const LEGACY_WORKER_POOLS_PATH: &str = "worker_pools.toml"; -/// Resolves the layered solver config: CLI flags > local config file > embedded default. +/// Overall time budget for the remote config fetch (including its internal retries); +/// startup resolves without the remote layer when it elapses. +const REMOTE_CONFIG_FETCH_TIMEOUT: Duration = Duration::from_secs(2); + +/// Resolves the layered solver config: +/// CLI flags > local config file > remote config (S3) > embedded default. /// /// Local files fail fast when explicitly passed or malformed — a broken local setup should -/// not start silently misconfigured. -fn resolve_solver_config(args: &cli::ServeArgs) -> Result { +/// not start silently misconfigured. The remote fetch never fails startup: on any error or +/// timeout a warning is logged and the lower layers apply unchanged. +async fn resolve_solver_config( + args: &cli::ServeArgs, +) -> Result { if args.worker_pools_config.is_some() { warn!( "--worker-pools-config is deprecated and will be removed soon; move the [pools] \ @@ -278,9 +285,40 @@ fn resolve_solver_config(args: &cli::ServeArgs) -> Result { + info!(url, "fetched remote config"); + Some(partial) + } + Ok(Err(e)) => { + warn!(url, error = %e, "remote config fetch failed; continuing without it"); + None + } + Err(_elapsed) => { + warn!( + url, + timeout_ms = REMOTE_CONFIG_FETCH_TIMEOUT.as_millis() as u64, + "remote config fetch timed out; continuing without it" + ); + None + } + } + }; + + // Ascending priority: embedded default, then the remote config, then the local config + // file, then CLI overrides. let config = embedded_default() .clone() + .apply(&remote_config.unwrap_or_default()) .apply(&local_file.unwrap_or_default()) .apply(&overrides); config @@ -292,14 +330,13 @@ fn resolve_solver_config(args: &cli::ServeArgs) -> Result Result { - let chain = parse_chain(&args.chain) - .map_err(|e| SolverError::SetupError(format!("failed to parse chain: {}", e)))?; + let chain = args.chain; - let config = resolve_solver_config(args)?; + let config = resolve_solver_config(args).await?; info!(?config, "solver config resolved"); - let tycho_url = resolve_tycho_url(&args.chain, args.tycho_url.as_deref())?; - let rpc_url = resolve_rpc_url(&args.chain, args.rpc_url.as_deref())?; + let tycho_url = resolve_tycho_url(&chain.to_string(), args.tycho_url.as_deref())?; + let rpc_url = resolve_rpc_url(&chain.to_string(), args.rpc_url.as_deref())?; let protocols = resolve_protocols( &tycho_url,