From 9e0ee8a08bb4b5b53e511bbfa4c2707897150ff2 Mon Sep 17 00:00:00 2001 From: Zheng Lu Date: Fri, 18 Sep 2026 00:40:20 +0100 Subject: [PATCH 1/2] feat(tool): add SearXNG as a selectable web_search provider (#326) Add a keyless, self-hosted SearXNG backend for the gateway-owned web_search tool with a mandatory base URL validated at startup, typed request/response mapping (categories split, time_range, language normalization, numeric safesearch), client-side domain filtering and count truncation, actionable 403/429 diagnostics for the JSON format and bot-detection limiter, hermetic Axum mock coverage, and documentation. Signed-off-by: Zheng Lu --- ARCHITECTURE.md | 4 +- CHANGELOG.md | 13 + README.md | 57 +- clippy.toml | 2 + crates/agentic-server-core/src/config.rs | 31 +- crates/agentic-server-core/src/tool/mod.rs | 1 + .../src/tool/web_search/args.rs | 4 +- .../src/tool/web_search/mod.rs | 59 +- .../src/tool/web_search/provider.rs | 41 +- .../src/tool/web_search/searxng.rs | 724 ++++++++++++++++++ .../tests/web_search_searxng_test.rs | 689 +++++++++++++++++ crates/agentic-server/src/config_file.rs | 19 +- .../agentic-server/src/web_search_config.rs | 111 ++- docs/deploying/README.md | 14 + docs/deploying/kubernetes.md | 14 +- 15 files changed, 1717 insertions(+), 66 deletions(-) create mode 100644 crates/agentic-server-core/src/tool/web_search/searxng.rs create mode 100644 crates/agentic-server-core/tests/web_search_searxng_test.rs diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 43787759..21930d73 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -951,8 +951,8 @@ declaration until they have a complete handler and execution path. (`CodexNamespaceHandler`), and `tool_search.rs` (`ToolSearchHandler`). Their calls are returned for the client to resolve; the gateway does not execute them. - **Gateway-owned / built-in** tools implement both traits: see `web_search/mod.rs` - (`WebSearchHandler`, backed by the configured `WebSearchProvider` in `web_search/you.rs` - or `web_search/brave.rs`) and `mcp/handler.rs` (`McpHandler`, backed + (`WebSearchHandler`, backed by the configured `WebSearchProvider` in `web_search/you.rs`, + `web_search/brave.rs`, or `web_search/searxng.rs`) and `mcp/handler.rs` (`McpHandler`, backed by `mcp/client.rs`'s MCP protocol client and `mcp/pool.rs`'s connection pool). They have no client translator association because the gateway owns their execution and public lifecycle. diff --git a/CHANGELOG.md b/CHANGELOG.md index 53ba7b6e..8ca25e99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,19 @@ All notable changes to Agentic API are documented here. ### Added +- Added SearXNG as a selectable backend for the gateway-owned `web_search` tool (#326, Phase 3 of #291). Select it + with `AGENTIC_WEB_SEARCH_PROVIDER=searxng` or `[web_search] provider = "searxng"` and point + `AGENTIC_WEB_SEARCH_BASE_URL` or `[web_search] base_url` at a self-hosted instance; the endpoint is mandatory and + the server refuses to start without it. No API key is needed; `SEARXNG_API_KEY` (or the variable named by + `api_key_env`) is sent as a `Bearer` token only when set. Web and news results come from one + `format=json&categories=general,news` request per query, split by category. The gateway adapts the shared tool + contract: `allowed_domains` / `blocked_domains` and the model's `include_domains` / `exclude_domains` are enforced + client-side on a label boundary, `count` is applied client-side after filtering, `freshness` maps to `time_range` + (date ranges are ignored), `language` is normalized to SearXNG's `xx` / `xx-YY` form, `safesearch` maps to + `0` / `1` / `2`, and `country` plus the You.com-specific arguments are ignored. A `403` is reported as the JSON + format being disabled, and a `429` explains SearXNG's bot-detection limiter, which rejects the gateway's + `Accept-Encoding`-free requests unless its address is on `pass_ip`; neither is retried. Each SearXNG `metadata[]` + entry carries `"provider": "searxng"`. Concurrency inherits `max_concurrent_gateway_calls`. - Added typed per-model input-modality overrides to `config.toml` (`[models.""] input_modalities = ["text", "image"]`), validated at startup: unknown modality names, empty lists, duplicates, and image-only lists are rejected with the diff --git a/README.md b/README.md index 5567f928..96744fba 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ flowchart LR ## ✨ Key Features - 🔄 **Stateful conversations**: the server manages history via `previous_response_id`. No client-side message tracking, no replaying full transcripts. -- 🛠️ **Server-side tool execution**: an explicit tool-ownership model (gateway / client / provider) decides exactly what runs where. Web search ships today via [You.com](https://you.com) or [Brave Search](https://brave.com/search/api/), and the model executes multi-step tool chains automatically. +- 🛠️ **Server-side tool execution**: an explicit tool-ownership model (gateway / client / provider) decides exactly what runs where. Web search ships today via [You.com](https://you.com), [Brave Search](https://brave.com/search/api/), or a self-hosted [SearXNG](https://docs.searxng.org/) instance, and the model executes multi-step tool chains automatically. - 📡 **Every transport**: non-streaming HTTP, server-sent events for token streaming, and full **WebSocket** support for interactive clients. - 🧰 **Codex-ready**: accepts Codex-shaped Responses traffic out of the box, preserving the tool declarations and response item shapes Codex depends on. - 🏃 **Background execution**: fire-and-forget requests that keep processing server-side. @@ -193,6 +193,14 @@ AGENTIC_WEB_SEARCH_PROVIDER=brave BRAVE_API_KEY= \ cargo run -p agentic-server -- --llm-api-base http://0.0.0.0:5050 ``` +Running fully on-premise? Point the gateway at a self-hosted [SearXNG](https://docs.searxng.org/) instance instead; +no API key is needed, only its URL: + +```bash +AGENTIC_WEB_SEARCH_PROVIDER=searxng AGENTIC_WEB_SEARCH_BASE_URL=http://127.0.0.1:8080 \ + cargo run -p agentic-server -- --llm-api-base http://0.0.0.0:5050 +``` + The default database is `~/.agentic-api/agentic_api.db`, so running an installed binary does not create state in the current directory. Set `AGENTIC_API_HOME` to an absolute directory to move both the default database and user configuration, or set `DATABASE_URL`/`--db-url` to select a different database. @@ -226,12 +234,12 @@ llm_api_base = "http://127.0.0.1:5050" # database_url = "postgresql://agentic-api@localhost/agentic_api" [web_search] -# Search backend for the gateway-owned web_search tool: "you" (default) or "brave". +# Search backend for the gateway-owned web_search tool: "you" (default), "brave", or "searxng". provider = "you" base_url = "https://api.ydc-index.io" api_key_env = "YOU_API_KEY" # Concurrent provider requests inside one batched web-search call; unset uses -# the provider default (Brave: 1, You.com: max_concurrent_gateway_calls). +# the provider default (Brave: 1, You.com and SearXNG: max_concurrent_gateway_calls). # max_concurrent_queries = 1 [mcp] @@ -304,9 +312,9 @@ every provider. | Setting | Environment variable | `config.toml` key | Default | | :--- | :--- | :--- | :--- | | Provider | `AGENTIC_WEB_SEARCH_PROVIDER` | `[web_search] provider` | `you` | -| API key | variable named by `api_key_env` | `[web_search] api_key_env` | `YOU_API_KEY` / `BRAVE_API_KEY` | -| Endpoint | `AGENTIC_WEB_SEARCH_BASE_URL` (or `YOU_API_BASE_URL` for You.com) | `[web_search] base_url` | none for You.com; `https://api.search.brave.com` for Brave | -| Concurrent queries | `AGENTIC_WEB_SEARCH_MAX_CONCURRENT_QUERIES` | `[web_search] max_concurrent_queries` | You.com inherits `max_concurrent_gateway_calls`; Brave `1` | +| API key | variable named by `api_key_env` | `[web_search] api_key_env` | `YOU_API_KEY` / `BRAVE_API_KEY` / `SEARXNG_API_KEY` (optional) | +| Endpoint | `AGENTIC_WEB_SEARCH_BASE_URL` (or `YOU_API_BASE_URL` for You.com) | `[web_search] base_url` | none for You.com; `https://api.search.brave.com` for Brave; **required** for SearXNG | +| Concurrent queries | `AGENTIC_WEB_SEARCH_MAX_CONCURRENT_QUERIES` | `[web_search] max_concurrent_queries` | You.com and SearXNG inherit `max_concurrent_gateway_calls`; Brave `1` | **You.com** (`provider = "you"`) is the default and behaves exactly as before: domain filters are applied by the provider, `count` accepts 1–100, and the You.com-specific `livecrawl`, `livecrawl_formats`, `crawl_timeout`, and @@ -338,6 +346,41 @@ provider = "brave" api_key_env = "BRAVE_API_KEY" ``` +**SearXNG** (`provider = "searxng"`) runs against a [self-hosted SearXNG](https://docs.searxng.org/admin/installation.html) +instance, so prompts and search queries never leave your network and no API key is needed. The endpoint is +mandatory: the server refuses to start when `searxng` is selected without `AGENTIC_WEB_SEARCH_BASE_URL` or +`[web_search] base_url`. Two instance settings matter: + +- The JSON output format must be enabled: add `json` to `search.formats` in SearXNG's `settings.yml` + (`formats: [html, json]`). Without it SearXNG answers `403`, which the failed `web_search_call` explains. +- If the instance runs with `server.limiter: true` (the default in the official `searxng-docker` template), its bot + detection rejects requests that lack `Accept-Encoding: gzip`, which the gateway deliberately never sends. Add the + gateway's address to `botdetection.ip_lists.pass_ip` in `limiter.toml`, or disable the limiter for an internal + instance; otherwise every search fails with `429`. + +The gateway adapts the shared tool contract to SearXNG: + +- Web and news results come from one `categories=general,news` request per query, split by each hit's category. +- `allowed_domains` / `blocked_domains` (and the model's `include_domains` / `exclude_domains`) are enforced by the + gateway after the response arrives; `count` is applied by the gateway after filtering, since SearXNG has no + result-count parameter. Without `count` or `search_context_size` every hit the instance returned is passed on. +- `freshness` maps to `time_range=day|week|month|year`; a `YYYY-MM-DDtoYYYY-MM-DD` range has no SearXNG equivalent + and is ignored. `language` is normalized to SearXNG's `xx` / `xx-YY` form (`zh-Hans` becomes `zh`); `safesearch` + maps to `0` / `1` / `2`. +- `country` and the You.com-specific arguments are ignored (logged at debug level). +- Each per-query `metadata[]` entry carries `"provider": "searxng"`. +- Concurrency inherits `max_concurrent_gateway_calls`; lower `max_concurrent_queries` for a small instance. Rate + limits (`429`) fail that `web_search_call` without an automatic retry. + +If the instance sits behind an authenticating reverse proxy, set `SEARXNG_API_KEY` (or the variable named by +`api_key_env`) and the gateway sends it as a `Bearer` token. Example: + +```toml +[web_search] +provider = "searxng" +base_url = "http://searxng.internal:8080" +``` + Restrict the file to the service account (for example, `chmod 600 ~/.agentic-api/config.toml`), especially if you add credentialed `database_url`, MCP headers, or stdio MCP environment values. Prefer `DATABASE_URL`, referenced API-key environment variables, and a deployment secret manager for secrets. @@ -437,7 +480,7 @@ Claude Code's own tools (Bash, Edit, Read, …) stay **client-owned** — Claude Current Claude Code versions declare Anthropic's native `web_search_20250305` server tool. Agentic API translates that declaration for the upstream model and executes the resulting search server-side against the configured search backend -(You.com or Brave Search, see [Web search providers](#web-search-providers)); no MCP server or tool alias is required: +(You.com, Brave Search, or SearXNG, see [Web search providers](#web-search-providers)); no MCP server or tool alias is required: ```bash YOU_API_KEY= YOU_API_BASE_URL= \ diff --git a/clippy.toml b/clippy.toml index b339f7c3..276025eb 100644 --- a/clippy.toml +++ b/clippy.toml @@ -1 +1,3 @@ msrv = "1.85" +# Product names that are not Rust identifiers and need no backticks in docs. +doc-valid-idents = ["SearXNG", ".."] diff --git a/crates/agentic-server-core/src/config.rs b/crates/agentic-server-core/src/config.rs index fbfa1b87..3c7dfa95 100644 --- a/crates/agentic-server-core/src/config.rs +++ b/crates/agentic-server-core/src/config.rs @@ -101,11 +101,12 @@ pub enum WebSearchProviderKind { #[default] You, Brave, + Searxng, } impl WebSearchProviderKind { /// Every selectable provider, in the order operator-facing messages list them. - pub const ALL: [Self; 2] = [Self::You, Self::Brave]; + pub const ALL: [Self; 3] = [Self::You, Self::Brave, Self::Searxng]; /// Environment variable that conventionally carries this provider's API key. #[must_use] @@ -113,16 +114,18 @@ impl WebSearchProviderKind { match self { Self::You => "YOU_API_KEY", Self::Brave => "BRAVE_API_KEY", + Self::Searxng => "SEARXNG_API_KEY", } } /// Endpoint used when neither the environment nor the configuration file /// sets one. You.com has no default so a deployment that fails today keeps - /// failing the same way (#291 Q2). + /// failing the same way (#291 Q2); SearXNG is self-hosted, so its endpoint + /// is mandatory and never defaulted. #[must_use] pub const fn default_base_url(self) -> Option<&'static str> { match self { - Self::You => None, + Self::You | Self::Searxng => None, Self::Brave => Some("https://api.search.brave.com"), } } @@ -133,7 +136,7 @@ impl WebSearchProviderKind { #[must_use] pub const fn default_max_concurrent_queries(self) -> Option { match self { - Self::You => None, + Self::You | Self::Searxng => None, Self::Brave => Some(DEFAULT_BRAVE_MAX_CONCURRENT_QUERIES), } } @@ -144,15 +147,17 @@ impl WebSearchProviderKind { match self { Self::You => "You.com", Self::Brave => "Brave Search", + Self::Searxng => "SearXNG", } } - /// Configuration label (`you`, `brave`) matching the serialized form. + /// Configuration label (`you`, `brave`, `searxng`) matching the serialized form. #[must_use] pub const fn config_name(self) -> &'static str { match self { Self::You => "you", Self::Brave => "brave", + Self::Searxng => "searxng", } } @@ -450,6 +455,12 @@ mod tests { NonZeroUsize::new(1) ); assert!(!WebSearchProviderKind::Brave.is_you()); + + assert_eq!(WebSearchProviderKind::Searxng.to_string(), "SearXNG"); + assert_eq!(WebSearchProviderKind::Searxng.default_api_key_env(), "SEARXNG_API_KEY"); + assert_eq!(WebSearchProviderKind::Searxng.default_base_url(), None); + assert_eq!(WebSearchProviderKind::Searxng.default_max_concurrent_queries(), None); + assert!(!WebSearchProviderKind::Searxng.is_you()); } #[test] @@ -464,16 +475,24 @@ mod tests { "you".parse::().unwrap(), WebSearchProviderKind::You ); + assert_eq!( + " SearXNG ".parse::().unwrap(), + WebSearchProviderKind::Searxng + ); let error = "bing".parse::().unwrap_err(); assert_eq!( error.to_string(), - "unknown web_search provider \"bing\"; expected one of: you, brave" + "unknown web_search provider \"bing\"; expected one of: you, brave, searxng" ); assert_eq!( serde_json::to_string(&WebSearchProviderKind::Brave).unwrap(), "\"brave\"" ); + assert_eq!( + serde_json::to_string(&WebSearchProviderKind::Searxng).unwrap(), + "\"searxng\"" + ); assert_eq!( serde_json::from_str::("\"you\"").unwrap(), WebSearchProviderKind::You diff --git a/crates/agentic-server-core/src/tool/mod.rs b/crates/agentic-server-core/src/tool/mod.rs index 57caef1f..d0f8aca3 100644 --- a/crates/agentic-server-core/src/tool/mod.rs +++ b/crates/agentic-server-core/src/tool/mod.rs @@ -28,3 +28,4 @@ pub use shell::ShellHandler; pub(crate) use tool_search::ToolSearchMetadata; pub use tool_search::{ToolSearchHandler, ToolSearchState}; pub use web_search::WebSearchHandler; +pub use web_search::searxng::SEARXNG_BASE_URL_HINT; diff --git a/crates/agentic-server-core/src/tool/web_search/args.rs b/crates/agentic-server-core/src/tool/web_search/args.rs index 08278732..39029224 100644 --- a/crates/agentic-server-core/src/tool/web_search/args.rs +++ b/crates/agentic-server-core/src/tool/web_search/args.rs @@ -211,8 +211,8 @@ pub(crate) fn clean_vec(values: Option<&[String]>) -> Option> { /// (label boundary), compared case-insensitively after IDNA normalization. A /// URL without a parseable host cannot be checked, so it is rejected whenever /// any allowlist or blocklist is active (fail closed). You.com filters -/// server-side, so this is not applied on that path; Brave Search applies it -/// to every result section. +/// server-side, so this is not applied on that path; Brave Search and SearXNG +/// apply it to every result section. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub(crate) struct DomainFilter { include: Vec, diff --git a/crates/agentic-server-core/src/tool/web_search/mod.rs b/crates/agentic-server-core/src/tool/web_search/mod.rs index 94ce8d28..4a9b7105 100644 --- a/crates/agentic-server-core/src/tool/web_search/mod.rs +++ b/crates/agentic-server-core/src/tool/web_search/mod.rs @@ -2,12 +2,13 @@ //! //! `mod.rs` owns the OpenAI-facing adapter: the [`WebSearchHandler`], the //! mapping to public `web_search_call` output items. [`provider`] defines the -//! private provider contract and normalized result types. [`args`] parses the model's arguments; provider modules -//! ([`you`], [`brave`]) shape requests and map responses. +//! private provider contract, normalized result types, and shared response helpers. [`args`] parses the model's +//! arguments; provider modules ([`you`], [`brave`], [`searxng`]) shape requests and map responses. pub(crate) mod args; pub(crate) mod brave; mod provider; +pub(crate) mod searxng; pub(crate) mod you; use std::collections::HashMap; @@ -18,7 +19,7 @@ use std::pin::Pin; use std::sync::Arc; use futures::{StreamExt, TryStreamExt}; -use serde::{Deserialize, Deserializer, Serialize}; +use serde::Serialize; use serde_json::Value; use tokio::sync::Semaphore; @@ -27,6 +28,8 @@ use self::brave::BraveSearchProvider; use self::provider::{ ApiKey, WebSearchProvider, WebSearchProviderMetadata, WebSearchProviderResponse, WebSearchResult, clean_base_url, }; +pub(crate) use self::provider::{null_as_default, read_response_limited}; +use self::searxng::SearxngSearchProvider; use self::you::{YOU_API_BASE_URL, YOU_API_KEY, YouSearchProvider}; use super::handler::MAX_GATEWAY_TOOL_OUTPUT_BYTES; use super::handler::{GatewayExecutor, GatewayToolEventPlan, ToolError, ToolHandler, ToolOutput}; @@ -227,6 +230,12 @@ impl WebSearchHandler { .or(WebSearchProviderKind::Brave.default_max_concurrent_queries()) .unwrap_or(max_concurrent_gateway_calls), )), + WebSearchProviderKind::Searxng => Arc::new(SearxngSearchProvider::from_values( + client, + config.api_key.clone(), + config.base_url.clone(), + config.max_concurrent_queries.unwrap_or(max_concurrent_gateway_calls), + )), }; let effective = effective_query_concurrency(provider.as_ref(), requested); Self::with_provider_and_query_concurrency(provider, effective) @@ -357,38 +366,6 @@ struct WebSearchToolOutput<'a> { metadata: Vec, } -/// Deserializes an explicit JSON `null` as the field's default instead of -/// failing, so a degenerate provider response cannot fail the whole search. -pub(crate) fn null_as_default<'de, D, T>(deserializer: D) -> Result -where - D: Deserializer<'de>, - T: Default + Deserialize<'de>, -{ - Option::::deserialize(deserializer).map(Option::unwrap_or_default) -} - -/// Reads a provider HTTP response body, failing as soon as it exceeds -/// [`MAX_GATEWAY_TOOL_OUTPUT_BYTES`] so an oversized provider reply is never -/// buffered in full. Every provider module reads its responses through here. -pub(super) async fn read_response_limited( - resp: reqwest::Response, - provider: WebSearchProviderKind, -) -> Result { - let mut stream = resp.bytes_stream(); - let mut body = Vec::new(); - while let Some(chunk) = stream.next().await { - let chunk = chunk - .map_err(|error| ToolError::Execution(format!("failed to read {provider} search response: {error}")))?; - if chunk.len() > MAX_GATEWAY_TOOL_OUTPUT_BYTES.saturating_sub(body.len()) { - return Err(ToolError::Execution(format!( - "{provider} search response exceeded {MAX_GATEWAY_TOOL_OUTPUT_BYTES} bytes" - ))); - } - body.extend_from_slice(&chunk); - } - String::from_utf8(body).map_err(|_| ToolError::Execution(format!("{provider} search response was not valid UTF-8"))) -} - impl ToolHandler for WebSearchHandler { type ToolParams = WebSearchToolParam; @@ -804,8 +781,18 @@ mod tests { assert_eq!(handler.max_concurrent_queries.get(), 1); assert_eq!(handler.query_permits.available_permits(), 1); let raised = brave.with_max_concurrent_queries(NonZeroUsize::new(3)); - let handler = WebSearchHandler::from_config(client, &raised, gateway_limit); + let handler = WebSearchHandler::from_config(Arc::clone(&client), &raised, gateway_limit); assert_eq!(handler.max_concurrent_queries.get(), 3); + + // SearXNG is keyless and inherits the gateway limit unless the operator lowers it. + let searxng = WebSearchProviderConfig::new(None, Some("http://searxng:8080".to_owned())) + .with_provider(WebSearchProviderKind::Searxng); + let handler = WebSearchHandler::from_config(Arc::clone(&client), &searxng, gateway_limit); + assert!(format!("{handler:?}").contains("SearxngSearchProvider")); + assert_eq!(handler.max_concurrent_queries.get(), 5); + let lowered = searxng.with_max_concurrent_queries(NonZeroUsize::new(2)); + let handler = WebSearchHandler::from_config(client, &lowered, gateway_limit); + assert_eq!(handler.max_concurrent_queries.get(), 2); } #[test] diff --git a/crates/agentic-server-core/src/tool/web_search/provider.rs b/crates/agentic-server-core/src/tool/web_search/provider.rs index fc307dc0..73205757 100644 --- a/crates/agentic-server-core/src/tool/web_search/provider.rs +++ b/crates/agentic-server-core/src/tool/web_search/provider.rs @@ -1,16 +1,17 @@ -//! Shared search-provider contract and normalized result types. +//! Shared search-provider contract, normalized result types, and the response +//! helpers every provider module reads its upstream replies through. use std::fmt; use std::future::Future; use std::num::NonZeroUsize; use std::pin::Pin; -use serde::{Deserialize, Serialize}; +use futures::StreamExt; +use serde::{Deserialize, Deserializer, Serialize}; use super::args::WebSearchArguments; -use super::null_as_default; use crate::config::WebSearchProviderKind; -use crate::tool::handler::ToolError; +use crate::tool::handler::{MAX_GATEWAY_TOOL_OUTPUT_BYTES, ToolError}; use crate::types::tools::WebSearchToolParam; /// Provider credential whose `Debug` output never contains the secret. @@ -109,3 +110,35 @@ pub(crate) struct WebSearchProviderResponse { pub news: Vec, pub metadata: WebSearchProviderMetadata, } + +/// Deserializes an explicit JSON `null` as the field's default instead of +/// failing, so a degenerate provider response cannot fail the whole search. +pub(crate) fn null_as_default<'de, D, T>(deserializer: D) -> Result +where + D: Deserializer<'de>, + T: Default + Deserialize<'de>, +{ + Option::::deserialize(deserializer).map(Option::unwrap_or_default) +} + +/// Reads a provider HTTP response body, failing as soon as it exceeds +/// [`MAX_GATEWAY_TOOL_OUTPUT_BYTES`] so an oversized provider reply is never +/// buffered in full. Every provider module reads its responses through here. +pub(crate) async fn read_response_limited( + resp: reqwest::Response, + provider: WebSearchProviderKind, +) -> Result { + let mut stream = resp.bytes_stream(); + let mut body = Vec::new(); + while let Some(chunk) = stream.next().await { + let chunk = chunk + .map_err(|error| ToolError::Execution(format!("failed to read {provider} search response: {error}")))?; + if chunk.len() > MAX_GATEWAY_TOOL_OUTPUT_BYTES.saturating_sub(body.len()) { + return Err(ToolError::Execution(format!( + "{provider} search response exceeded {MAX_GATEWAY_TOOL_OUTPUT_BYTES} bytes" + ))); + } + body.extend_from_slice(&chunk); + } + String::from_utf8(body).map_err(|_| ToolError::Execution(format!("{provider} search response was not valid UTF-8"))) +} diff --git a/crates/agentic-server-core/src/tool/web_search/searxng.rs b/crates/agentic-server-core/src/tool/web_search/searxng.rs new file mode 100644 index 00000000..dfe01e61 --- /dev/null +++ b/crates/agentic-server-core/src/tool/web_search/searxng.rs @@ -0,0 +1,724 @@ +//! SearXNG provider for `web_search`. +//! +//! Owns request shaping against a self-hosted SearXNG instance's +//! `GET /search?format=json` and the mapping of its JSON envelope onto the +//! provider-neutral [`WebSearchProviderResponse`]. SearXNG is a keyless +//! metasearch engine, so it differs from You.com and Brave in ways the gateway +//! adapts here rather than surfacing to the model: +//! +//! - the endpoint is mandatory: there is no public SearXNG API, so a missing +//! base URL is a configuration error; +//! - web and news hits arrive in one `results[]` list, split by each hit's +//! `category` (`categories=general,news` is requested once per query); +//! - there is no server-side domain filter, so `include_domains` / +//! `exclude_domains` are applied client-side through [`DomainFilter`]; +//! - there is no `count` parameter, so results are truncated client-side after +//! filtering (paging through `pageno` is out of scope); +//! - `freshness` maps onto `time_range=day|week|month|year`; a date range has +//! no SearXNG equivalent and is ignored; +//! - `language` must match SearXNG's `xx` / `xx-YY` shape or the instance +//! answers `400`, so BCP 47 tags are normalized before sending; +//! - `safesearch` is an integer (`0`, `1`, `2`) rather than a name. +//! +//! `Accept-Encoding` is deliberately never sent: the core `reqwest` build has no +//! `gzip` feature, so a compressed body could not be decoded. SearXNG's optional +//! bot-detection limiter (`server.limiter: true`) rejects such requests with +//! `429`, which the error message explains to the operator. + +use std::future::Future; +use std::num::NonZeroUsize; +use std::pin::Pin; +use std::sync::Arc; + +use reqwest::StatusCode; +use serde::Deserialize; + +use super::args::{DomainFilter, Freshness, WebSearchArguments, clean_string, clean_vec, validate_count}; +use super::{ + ApiKey, WebSearchProvider, WebSearchProviderMetadata, WebSearchProviderResponse, WebSearchResult, clean_base_url, + null_as_default, read_response_limited, +}; +use crate::config::WebSearchProviderKind; +use crate::tool::handler::ToolError; +use crate::types::tools::{WebSearchContextSize, WebSearchToolParam}; + +pub(crate) const SEARXNG_API_KEY: &str = WebSearchProviderKind::Searxng.default_api_key_env(); + +/// Operator-facing fix for a missing SearXNG endpoint, shared by the startup +/// check in `agentic-server` and the execution-time fallback here. +pub const SEARXNG_BASE_URL_HINT: &str = "SearXNG requires a base URL; set AGENTIC_WEB_SEARCH_BASE_URL or [web_search] base_url \ + (for example http://searxng:8080)"; + +const SEARCH_PATH: &str = "/search"; +const CATEGORIES: &str = "general,news"; +const NEWS_CATEGORY: &str = "news"; + +#[derive(Debug, Clone)] +pub(crate) struct SearxngSearchProvider { + client: Arc, + api_key: Option, + base_url: Option, + max_concurrent_requests: NonZeroUsize, +} + +impl SearxngSearchProvider { + /// Builds a provider from optional environment-style values: a blank key + /// counts as unset (SearXNG needs none); a blank base URL counts as unset + /// and fails at execution time because SearXNG has no default endpoint. + pub(crate) fn from_values( + client: Arc, + api_key: Option, + base_url: Option, + max_concurrent_requests: NonZeroUsize, + ) -> Self { + Self { + client, + api_key: api_key + .map(|value| value.trim().to_owned()) + .filter(|value| !value.is_empty()) + .map(ApiKey), + base_url: base_url.and_then(|value| clean_base_url(&value)), + max_concurrent_requests, + } + } +} + +impl WebSearchProvider for SearxngSearchProvider { + fn search<'a>( + &'a self, + query: &'a str, + args: &'a WebSearchArguments, + config: &'a WebSearchToolParam, + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + let base_url = self + .base_url + .as_deref() + .ok_or_else(|| ToolError::Config(SEARXNG_BASE_URL_HINT.to_owned()))?; + let request = SearxngSearchRequest::from_args_and_config(query, args, config)?; + let mut builder = self + .client + .get(format!("{base_url}{SEARCH_PATH}")) + .query(&request.query_params()) + .header("Accept", "application/json"); + if let Some(api_key) = &self.api_key { + builder = builder.bearer_auth(&api_key.0); + } + let resp = builder + .send() + .await + .map_err(|e| ToolError::Execution(format!("SearXNG request failed: {e}")))?; + + let status = resp.status(); + if !status.is_success() { + return Err(failure_from_status(resp).await); + } + + let response_text = read_response_limited(resp, WebSearchProviderKind::Searxng).await?; + let response: SearxngSearchResponse = serde_json::from_str(&response_text).map_err(|e| { + ToolError::Execution(format!( + "SearXNG returned a non-JSON response ({e}); confirm base_url points at a SearXNG instance \ + and that `search.formats` in settings.yml includes `json`" + )) + })?; + Ok(response.into_provider_response(&request)) + }) + } + + fn max_concurrent_requests(&self) -> Option { + Some(self.max_concurrent_requests) + } +} + +/// Maps a non-2xx SearXNG response to an actionable, credential-free error. +/// +/// SearXNG answers `403` when `format=json` is not enabled, so that status is +/// a configuration hint before it is an authentication one. `429` is what the +/// bot-detection limiter returns for a request without `Accept-Encoding: +/// gzip`, which this client cannot send; it is reported without retrying. +async fn failure_from_status(resp: reqwest::Response) -> ToolError { + let status = resp.status(); + match status { + StatusCode::FORBIDDEN => ToolError::Execution(format!( + "SearXNG refused the request ({status}); enable the JSON format with `search.formats: [html, json]` \ + in settings.yml, or check {SEARXNG_API_KEY} if the instance is behind an authenticating proxy" + )), + StatusCode::UNAUTHORIZED => ToolError::Execution(format!( + "SearXNG rejected the credential ({status}); check {SEARXNG_API_KEY}" + )), + StatusCode::TOO_MANY_REQUESTS => { + let hint = resp + .headers() + .get("retry-after") + .and_then(|value| value.to_str().ok()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map_or_else(String::new, |value| format!("; retry after {value}")); + ToolError::Execution(format!( + "SearXNG rate limited the request ({status}); the gateway does not retry{hint}. If the instance runs \ + with `server.limiter: true`, its bot detection blocks the gateway (which cannot send \ + Accept-Encoding: gzip): add the gateway address to `botdetection.ip_lists.pass_ip` in limiter.toml \ + or disable the limiter" + )) + } + StatusCode::BAD_REQUEST => { + let body = read_response_limited(resp, WebSearchProviderKind::Searxng) + .await + .unwrap_or_default(); + ToolError::Execution(format!("SearXNG rejected a search parameter ({status}): {body}")) + } + _ => { + let body = read_response_limited(resp, WebSearchProviderKind::Searxng) + .await + .unwrap_or_default(); + ToolError::Execution(format!("SearXNG returned {status}: {body}")) + } + } +} + +/// Query parameters for SearXNG's `GET /search`, derived from the model's +/// arguments and the request-level tool configuration, plus the client-side +/// adaptations SearXNG cannot apply itself. +#[derive(Debug, PartialEq, Eq)] +struct SearxngSearchRequest { + query: String, + /// Per-section result ceiling applied after filtering. + count: Option, + time_range: Option<&'static str>, + language: Option, + safesearch: Option, + domain_filter: DomainFilter, +} + +impl SearxngSearchRequest { + fn query_params(&self) -> Vec<(String, String)> { + let mut params = vec![ + ("q".to_owned(), self.query.clone()), + ("format".to_owned(), "json".to_owned()), + ("categories".to_owned(), CATEGORIES.to_owned()), + ]; + if let Some(time_range) = self.time_range { + params.push(("time_range".to_owned(), time_range.to_owned())); + } + if let Some(language) = &self.language { + params.push(("language".to_owned(), language.clone())); + } + if let Some(safesearch) = self.safesearch { + params.push(("safesearch".to_owned(), safesearch.to_string())); + } + params + } + + fn from_args_and_config( + query: &str, + args: &WebSearchArguments, + config: &WebSearchToolParam, + ) -> Result { + let count = args + .count + .or_else(|| { + config + .search_context_size + .map(WebSearchContextSize::default_count) + .map(u16::from) + }) + .map(validate_count) + .transpose()?; + let config_domains = config + .filters + .as_ref() + .and_then(|filters| clean_vec(filters.allowed_domains.as_deref())); + let config_blocked_domains = config + .filters + .as_ref() + .and_then(|filters| clean_vec(filters.blocked_domains.as_deref())); + let include_domains = config_domains.or_else(|| args.include_domains.clone()); + let exclude_domains = config_blocked_domains.or_else(|| args.exclude_domains.clone()); + if include_domains.is_some() && (exclude_domains.is_some() || args.boost_domains.is_some()) { + return Err(ToolError::Config( + "include_domains cannot be combined with exclude_domains or boost_domains".to_owned(), + )); + } + log_ignored_arguments(args, config); + + Ok(Self { + query: query.trim().to_owned(), + count, + time_range: args.freshness.and_then(searxng_time_range), + language: args.language.as_deref().and_then(searxng_language), + safesearch: args.safesearch.as_deref().and_then(searxng_safesearch), + domain_filter: DomainFilter::new(include_domains.as_deref(), exclude_domains.as_deref()), + }) + } +} + +/// Arguments without a SearXNG equivalent are dropped: the You.com-specific +/// crawl controls, `boost_domains` (no filtering semantics), and `country` +/// (SearXNG localizes through `language` only). +fn log_ignored_arguments(args: &WebSearchArguments, config: &WebSearchToolParam) { + let country = args.country.is_some() + || config + .user_location + .as_ref() + .is_some_and(|location| clean_string(location.country.as_deref()).is_some()); + let ignored: Vec<&str> = [ + ("livecrawl", args.livecrawl.is_some()), + ("livecrawl_formats", args.livecrawl_formats.is_some()), + ("crawl_timeout", args.crawl_timeout.is_some()), + ("boost_domains", args.boost_domains.is_some()), + ("country", country), + ] + .into_iter() + .filter_map(|(name, present)| present.then_some(name)) + .collect(); + if !ignored.is_empty() { + tracing::debug!(arguments = ?ignored, "ignored web_search arguments without a SearXNG equivalent"); + } +} + +/// Renders the typed freshness filter as SearXNG's `time_range`. SearXNG has no +/// date-range filter, so a range is dropped rather than approximated. +fn searxng_time_range(freshness: Freshness) -> Option<&'static str> { + match freshness { + Freshness::Day => Some("day"), + Freshness::Week => Some("week"), + Freshness::Month => Some("month"), + Freshness::Year => Some("year"), + Freshness::Range { .. } => { + tracing::debug!("ignored web_search freshness date range; SearXNG supports day/week/month/year only"); + None + } + } +} + +/// Normalizes a BCP 47 tag to the `xx` / `xx-YY` shape SearXNG validates +/// (`^[a-z]{2,3}(-[a-zA-Z]{2})?$`); `auto` and `all` pass through. Script and +/// variant subtags would make SearXNG answer `400`, so they are dropped, and a +/// tag with no usable primary subtag is ignored. +fn searxng_language(value: &str) -> Option { + let value = value.trim(); + if value.eq_ignore_ascii_case("auto") || value.eq_ignore_ascii_case("all") { + return Some(value.to_ascii_lowercase()); + } + let mut subtags = value.split(['-', '_']); + let primary = subtags.next()?.to_ascii_lowercase(); + if !(2..=3).contains(&primary.len()) || !primary.bytes().all(|byte| byte.is_ascii_lowercase()) { + tracing::debug!( + language = value, + "ignored web_search language that SearXNG cannot parse" + ); + return None; + } + let region = subtags.find(|subtag| subtag.len() == 2 && subtag.bytes().all(|byte| byte.is_ascii_alphabetic())); + Some(region.map_or(primary.clone(), |region| { + format!("{primary}-{}", region.to_ascii_uppercase()) + })) +} + +/// Maps the named `safesearch` levels onto SearXNG's `0` / `1` / `2`. +fn searxng_safesearch(value: &str) -> Option { + match value.trim().to_ascii_lowercase().as_str() { + "off" | "0" => Some(0), + "moderate" | "1" => Some(1), + "strict" | "2" => Some(2), + other => { + tracing::debug!( + safesearch = other, + "ignored web_search safesearch level unknown to SearXNG" + ); + None + } + } +} + +/// SearXNG's `format=json` envelope. Only `results` is modeled; `answers`, +/// `infoboxes`, `suggestions`, `corrections`, `unresponsive_engines`, and +/// unknown keys are ignored so upstream additions never break the provider. +#[derive(Debug, Default, Deserialize)] +struct SearxngSearchResponse { + #[serde(default, deserialize_with = "null_as_default")] + results: Vec, +} + +/// One SearXNG hit. `content` is the snippet, `publishedDate` the ISO +/// timestamp SearXNG serializes for dated results, and `pubdate` its +/// preformatted form; ranking and cosmetic fields (`engine`, `engines`, +/// `score`, `positions`, `thumbnail`, `parsed_url`) are not modeled. +#[derive(Debug, Default, Deserialize)] +struct SearxngResult { + #[serde(default)] + url: String, + #[serde(default)] + title: Option, + #[serde(default)] + content: Option, + #[serde(default)] + category: Option, + #[serde(default, rename = "publishedDate")] + published_date: Option, + #[serde(default)] + pubdate: Option, +} + +impl SearxngResult { + fn is_news(&self) -> bool { + self.category + .as_deref() + .is_some_and(|category| category.trim().eq_ignore_ascii_case(NEWS_CATEGORY)) + } +} + +impl From for WebSearchResult { + fn from(result: SearxngResult) -> Self { + Self { + url: result.url.trim().to_owned(), + title: clean_string(result.title.as_deref()), + description: clean_string(result.content.as_deref()), + snippets: Vec::new(), + page_age: clean_string(result.published_date.as_deref()) + .or_else(|| clean_string(result.pubdate.as_deref())), + contents: None, + } + } +} + +impl SearxngSearchResponse { + fn into_provider_response(self, request: &SearxngSearchRequest) -> WebSearchProviderResponse { + let (news, web): (Vec, Vec) = + self.results.into_iter().partition(SearxngResult::is_news); + let mut web: Vec = web.into_iter().map(Into::into).collect(); + let mut news: Vec = news.into_iter().map(Into::into).collect(); + request.domain_filter.retain(&mut web); + request.domain_filter.retain(&mut news); + if let Some(count) = request.count { + web.truncate(usize::from(count)); + news.truncate(usize::from(count)); + } + WebSearchProviderResponse { + web, + news, + metadata: WebSearchProviderMetadata { + provider: WebSearchProviderKind::Searxng, + query: request.query.clone(), + search_uuid: None, + latency: None, + }, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::tools::{WebSearchFilters, WebSearchUserLocation}; + + fn build_provider(api_key: Option<&str>, base_url: Option<&str>) -> SearxngSearchProvider { + SearxngSearchProvider::from_values( + Arc::new(reqwest::Client::new()), + api_key.map(str::to_owned), + base_url.map(str::to_owned), + NonZeroUsize::new(3).unwrap(), + ) + } + + fn args(json: &str) -> WebSearchArguments { + WebSearchArguments::from_json(json).unwrap() + } + + fn request(json: &str, config: &WebSearchToolParam) -> SearxngSearchRequest { + SearxngSearchRequest::from_args_and_config("q", &args(json), config).unwrap() + } + + #[test] + fn provider_debug_is_redacted_and_base_url_has_no_default() { + let provider = build_provider(Some("super-secret-key"), Some(" http://searxng:8080/// ")); + let rendered = format!("{provider:?}"); + assert!(!rendered.contains("super-secret-key")); + assert!(rendered.contains("ApiKey()")); + assert_eq!(provider.base_url.as_deref(), Some("http://searxng:8080")); + assert_eq!(provider.max_concurrent_requests(), NonZeroUsize::new(3)); + + let provider = build_provider(Some(" "), Some(" ")); + assert!(provider.api_key.is_none()); + assert!(provider.base_url.is_none()); + assert!(build_provider(None, None).base_url.is_none()); + } + + #[tokio::test] + async fn search_without_base_url_names_the_setting() { + let provider = build_provider(None, None); + let error = provider + .search("q", &args(r#"{"query":"q"}"#), &WebSearchToolParam::default()) + .await + .unwrap_err(); + assert_eq!( + error.to_string(), + format!("invalid tool config: {SEARXNG_BASE_URL_HINT}") + ); + } + + #[test] + fn request_renders_every_argument_in_searxng_syntax() { + let request = SearxngSearchRequest::from_args_and_config( + " rust async ", + &args(r#"{"query":"rust async","count":7,"freshness":"week","language":"en-GB","safesearch":"strict"}"#), + &WebSearchToolParam::default(), + ) + .unwrap(); + assert_eq!( + request.query_params(), + [ + ("q", "rust async"), + ("format", "json"), + ("categories", "general,news"), + ("time_range", "week"), + ("language", "en-GB"), + ("safesearch", "2"), + ] + .map(|(key, value)| (key.to_owned(), value.to_owned())) + ); + assert_eq!(request.count, Some(7)); + assert!(request.domain_filter.is_empty()); + } + + #[test] + fn request_applies_context_size_default_and_validates_count() { + let config = WebSearchToolParam { + search_context_size: Some(WebSearchContextSize::Low), + ..WebSearchToolParam::default() + }; + assert_eq!( + request(r#"{"query":"q"}"#, &config).count.map(u16::from), + Some(u16::from(WebSearchContextSize::Low.default_count())) + ); + assert_eq!(request(r#"{"query":"q"}"#, &WebSearchToolParam::default()).count, None); + assert_eq!( + request(r#"{"query":"q","count":100}"#, &WebSearchToolParam::default()).count, + Some(100) + ); + let error = SearxngSearchRequest::from_args_and_config( + "q", + &args(r#"{"query":"q","count":101}"#), + &WebSearchToolParam::default(), + ) + .unwrap_err(); + assert_eq!( + error.to_string(), + "invalid tool config: web_search count must be between 1 and 100" + ); + } + + #[test] + fn freshness_maps_named_ranges_and_drops_date_ranges() { + assert_eq!(searxng_time_range(Freshness::Day), Some("day")); + assert_eq!(searxng_time_range(Freshness::Week), Some("week")); + assert_eq!(searxng_time_range(Freshness::Month), Some("month")); + assert_eq!(searxng_time_range(Freshness::Year), Some("year")); + let range: Freshness = "2026-01-02to2026-02-03".parse().unwrap(); + assert_eq!(searxng_time_range(range), None); + } + + #[test] + fn language_is_normalized_to_searxng_shape() { + assert_eq!(searxng_language("en").as_deref(), Some("en")); + assert_eq!(searxng_language(" EN-gb ").as_deref(), Some("en-GB")); + assert_eq!(searxng_language("pt_BR").as_deref(), Some("pt-BR")); + assert_eq!(searxng_language("zh-Hans").as_deref(), Some("zh")); + assert_eq!(searxng_language("zh-Hant-TW").as_deref(), Some("zh-TW")); + assert_eq!(searxng_language("ast").as_deref(), Some("ast")); + assert_eq!(searxng_language("Auto").as_deref(), Some("auto")); + assert_eq!(searxng_language("all").as_deref(), Some("all")); + assert_eq!(searxng_language("x"), None); + assert_eq!(searxng_language("english"), None); + assert_eq!(searxng_language("e1"), None); + } + + #[test] + fn safesearch_maps_named_levels_and_digits() { + assert_eq!(searxng_safesearch("off"), Some(0)); + assert_eq!(searxng_safesearch(" Moderate "), Some(1)); + assert_eq!(searxng_safesearch("strict"), Some(2)); + assert_eq!(searxng_safesearch("1"), Some(1)); + assert_eq!(searxng_safesearch("extreme"), None); + } + + #[test] + fn request_ignores_unsupported_arguments_and_builds_domain_filter() { + let config = WebSearchToolParam { + user_location: Some(WebSearchUserLocation { + country: Some("us".to_owned()), + ..WebSearchUserLocation::default() + }), + ..WebSearchToolParam::default() + }; + let request = request( + r#"{"query":"q","country":"de","livecrawl":"web","livecrawl_formats":["markdown"],"crawl_timeout":5,"boost_domains":["x.org"],"freshness":"2026-01-02to2026-02-03","language":"english","safesearch":"extreme","exclude_domains":["Example.com"]}"#, + &config, + ); + assert_eq!( + request.query_params(), + [("q", "q"), ("format", "json"), ("categories", "general,news")] + .map(|(key, value)| (key.to_owned(), value.to_owned())) + ); + assert!(!request.domain_filter.allows("https://docs.example.com/x")); + assert!(request.domain_filter.allows("https://other.org/x")); + } + + #[test] + fn request_prefers_tool_config_filters_and_rejects_conflicting_lists() { + let config = WebSearchToolParam { + filters: Some(WebSearchFilters { + allowed_domains: Some(vec!["rust-lang.org".to_owned()]), + blocked_domains: None, + }), + ..WebSearchToolParam::default() + }; + let request = request(r#"{"query":"q","include_domains":["example.com"]}"#, &config); + assert!(request.domain_filter.allows("https://doc.rust-lang.org/book")); + assert!(!request.domain_filter.allows("https://example.com/")); + + let error = SearxngSearchRequest::from_args_and_config( + "q", + &args(r#"{"query":"q","include_domains":["a.com"],"exclude_domains":["b.com"]}"#), + &WebSearchToolParam::default(), + ) + .unwrap_err(); + assert_eq!( + error.to_string(), + "invalid tool config: include_domains cannot be combined with exclude_domains or boost_domains" + ); + } + + #[test] + fn response_splits_sections_by_category_and_tolerates_unknown_fields() { + let response: SearxngSearchResponse = serde_json::from_str( + r#"{ + "query": "rust", + "number_of_results": 0, + "answers": [], "corrections": [], "infoboxes": [], "suggestions": [], + "unresponsive_engines": [["bing", "timeout"]], + "results": [ + { + "url": " https://www.rust-lang.org/ ", + "title": " Rust ", + "content": "A language", + "engine": "duckduckgo", + "engines": ["duckduckgo", "google"], + "parsed_url": ["https", "www.rust-lang.org", "/", "", "", ""], + "template": "default.html", + "positions": [1, 2], + "score": 3.5, + "category": "general", + "thumbnail": "" + }, + {"url": "https://example.com/no-title", "title": "", "content": null, "category": "general"}, + { + "url": "https://blog.rust-lang.org/1", + "title": "Release", + "content": "Rust 1.99 is out", + "category": "news", + "publishedDate": "2026-09-01T10:00:00", + "pubdate": "2026-09-01 10:00:00" + }, + {"url": "https://news.example.com/2", "title": "Dated", "category": "News", "pubdate": "2026-09-02 08:00:00"}, + {"url": "https://uncategorized.example.com/3"} + ] + }"#, + ) + .unwrap(); + let request = request(r#"{"query":"rust"}"#, &WebSearchToolParam::default()); + let response = response.into_provider_response(&request); + assert_eq!( + response.web, + vec![ + WebSearchResult { + url: "https://www.rust-lang.org/".to_owned(), + title: Some("Rust".to_owned()), + description: Some("A language".to_owned()), + ..WebSearchResult::default() + }, + WebSearchResult { + url: "https://example.com/no-title".to_owned(), + ..WebSearchResult::default() + }, + WebSearchResult { + url: "https://uncategorized.example.com/3".to_owned(), + ..WebSearchResult::default() + }, + ] + ); + assert_eq!( + response.news, + vec![ + WebSearchResult { + url: "https://blog.rust-lang.org/1".to_owned(), + title: Some("Release".to_owned()), + description: Some("Rust 1.99 is out".to_owned()), + page_age: Some("2026-09-01T10:00:00".to_owned()), + ..WebSearchResult::default() + }, + WebSearchResult { + url: "https://news.example.com/2".to_owned(), + title: Some("Dated".to_owned()), + page_age: Some("2026-09-02 08:00:00".to_owned()), + ..WebSearchResult::default() + }, + ] + ); + assert_eq!( + response.metadata, + WebSearchProviderMetadata { + provider: WebSearchProviderKind::Searxng, + query: "q".to_owned(), + search_uuid: None, + latency: None, + } + ); + assert_eq!( + serde_json::to_string(&response.metadata).unwrap(), + r#"{"provider":"searxng","query":"q"}"# + ); + } + + #[test] + fn response_tolerates_missing_and_null_results() { + let request = request(r#"{"query":"q"}"#, &WebSearchToolParam::default()); + for body in ["{}", r#"{"results":null}"#, r#"{"results":[]}"#] { + let response: SearxngSearchResponse = serde_json::from_str(body).unwrap(); + let response = response.into_provider_response(&request); + assert!(response.web.is_empty()); + assert!(response.news.is_empty()); + } + } + + #[test] + fn response_filters_domains_then_truncates_to_count() { + let response: SearxngSearchResponse = serde_json::from_str( + r#"{"results": [ + {"url": "https://docs.example.com/a"}, + {"url": "https://notexample.com/b"}, + {"url": "https://EXAMPLE.COM./c"}, + {"url": "not a url"}, + {"url": "https://api.example.com/d"}, + {"url": "https://news.example.com/e", "category": "news"}, + {"url": "https://other.org/f", "category": "news"}, + {"url": "https://feed.example.com/g", "category": "news"} + ]}"#, + ) + .unwrap(); + let request = request( + r#"{"query":"q","count":2,"include_domains":["example.com"]}"#, + &WebSearchToolParam::default(), + ); + let response = response.into_provider_response(&request); + let urls = |results: &[WebSearchResult]| results.iter().map(|r| r.url.clone()).collect::>(); + assert_eq!( + urls(&response.web), + ["https://docs.example.com/a", "https://EXAMPLE.COM./c"] + ); + assert_eq!( + urls(&response.news), + ["https://news.example.com/e", "https://feed.example.com/g"] + ); + } +} diff --git a/crates/agentic-server-core/tests/web_search_searxng_test.rs b/crates/agentic-server-core/tests/web_search_searxng_test.rs new file mode 100644 index 00000000..3146c8f5 --- /dev/null +++ b/crates/agentic-server-core/tests/web_search_searxng_test.rs @@ -0,0 +1,689 @@ +//! SearXNG provider behavior against a local Axum mock (#326). +//! +//! Every test binds its own `127.0.0.1:0` listener; nothing here reaches the +//! network. Mock handlers use `try_send` so a slow test never blocks inside +//! the server task. + +use std::num::NonZeroUsize; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; + +use agentic_core::config::{WebSearchProviderConfig, WebSearchProviderKind}; +use agentic_core::tool::{GatewayExecutor, SEARXNG_BASE_URL_HINT, WebSearchHandler}; +use agentic_core::types::event::MessageStatus; +use agentic_core::types::io::OutputItem; +use agentic_core::types::io::output::{FunctionToolCall, WebSearchCallStatus}; +use agentic_core::types::tools::{WebSearchContextSize, WebSearchFilters, WebSearchToolParam}; +use axum::extract::State; +use axum::http::{HeaderMap, StatusCode, Uri, header}; +use axum::response::IntoResponse; +use axum::routing::get; +use axum::{Json, Router}; +use tokio::net::TcpListener; +use tokio::sync::mpsc; + +mod support; + +const SEARXNG_SEARCH_PATH: &str = "/search"; +const GATEWAY_LIMIT: NonZeroUsize = NonZeroUsize::new(5).expect("nonzero gateway limit"); + +#[derive(Debug)] +struct CapturedSearxngRequest { + authorization: Option, + accept: Option, + accept_encoding: Option, + params: serde_json::Value, +} + +/// Response body served by the mock: JSON like a healthy instance, or raw +/// text with an explicit content type like a misconfigured one. +#[derive(Clone)] +enum MockBody { + Json(serde_json::Value), + Raw { + content_type: &'static str, + body: &'static str, + }, +} + +#[derive(Clone)] +struct MockSearxng { + tx: mpsc::Sender, + status: StatusCode, + headers: Vec<(&'static str, &'static str)>, + body: MockBody, +} + +fn capture(headers: &HeaderMap, uri: &Uri) -> CapturedSearxngRequest { + let header = |name: &str| { + headers + .get(name) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned) + }; + CapturedSearxngRequest { + authorization: header("authorization"), + accept: header("accept"), + accept_encoding: header("accept-encoding"), + params: support::query_params_as_json(uri), + } +} + +async fn spawn_mock_searxng( + status: StatusCode, + headers: Vec<(&'static str, &'static str)>, + body: MockBody, +) -> ( + String, + mpsc::Receiver, + tokio::task::JoinHandle<()>, +) { + let (tx, rx) = mpsc::channel(16); + let app = Router::new() + .route( + SEARXNG_SEARCH_PATH, + get( + |State(mock): State, headers: HeaderMap, uri: Uri| async move { + mock.tx + .try_send(capture(&headers, &uri)) + .expect("test channel has capacity"); + let mut response = match mock.body { + MockBody::Json(body) => (mock.status, Json(body)).into_response(), + MockBody::Raw { content_type, body } => { + (mock.status, [(header::CONTENT_TYPE, content_type)], body).into_response() + } + }; + for (name, value) in mock.headers { + response.headers_mut().insert(name, value.parse().unwrap()); + } + response + }, + ), + ) + .with_state(MockSearxng { + tx, + status, + headers, + body, + }); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let handle = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + (format!("http://{addr}"), rx, handle) +} + +async fn spawn_mock_json( + status: StatusCode, + body: serde_json::Value, +) -> ( + String, + mpsc::Receiver, + tokio::task::JoinHandle<()>, +) { + spawn_mock_searxng(status, Vec::new(), MockBody::Json(body)).await +} + +fn searxng_handler( + base_url: Option<&str>, + api_key: Option<&str>, + max_concurrent_queries: Option, +) -> WebSearchHandler { + let config = WebSearchProviderConfig::new(api_key.map(str::to_owned), base_url.map(str::to_owned)) + .with_provider(WebSearchProviderKind::Searxng) + .with_max_concurrent_queries(max_concurrent_queries); + WebSearchHandler::from_config(Arc::new(reqwest::Client::new()), &config, GATEWAY_LIMIT) +} + +/// A representative `format=json` envelope: ranking and cosmetic fields are +/// present so the test proves they are tolerated and dropped. +fn mixed_response() -> serde_json::Value { + serde_json::json!({ + "query": "rust async", + "number_of_results": 0, + "results": [ + { + "url": "https://example.com/rust", + "title": "Rust async guide", + "content": "A useful guide", + "engine": "duckduckgo", + "engines": ["duckduckgo", "brave"], + "parsed_url": ["https", "example.com", "/rust", "", "", ""], + "template": "default.html", + "positions": [1, 3], + "score": 2.5, + "category": "general", + "thumbnail": "https://example.com/rust.png" + }, + { + "url": "https://docs.example.org/tokio", + "title": "Tokio", + "content": "Runtime", + "category": "general" + }, + { + "url": "https://news.example.com/async-release", + "title": "Async release", + "content": "Released today", + "engine": "wikinews", + "category": "news", + "publishedDate": "2026-09-01T10:00:00", + "pubdate": "2026-09-01 10:00:00" + } + ], + "answers": [], + "corrections": [], + "infoboxes": [], + "suggestions": ["rust async book"], + "unresponsive_engines": [["bing", "timeout"]] + }) +} + +fn call(arguments: &str) -> FunctionToolCall { + FunctionToolCall { + id: "fc_searxng".to_owned(), + call_id: "call_searxng".to_owned(), + name: "web_search".to_owned(), + namespace: None, + arguments: arguments.to_owned(), + status: MessageStatus::Completed, + } +} + +async fn execute(handler: &WebSearchHandler, arguments: &str, params: &WebSearchToolParam) -> Result { + handler + .execute("call_searxng", "web_search", arguments, params) + .await + .map(|output| output.output) + .map_err(|error| error.to_string()) +} + +#[tokio::test] +async fn searxng_handler_maps_web_and_news_results_and_public_sources() { + let (base_url, mut captured, _handle) = spawn_mock_json(StatusCode::OK, mixed_response()).await; + let handler = searxng_handler(Some(&base_url), None, None); + let params = WebSearchToolParam::default(); + let arguments = r#"{"query":"rust async","count":50,"freshness":"week","country":"gb","language":"en-GB","safesearch":"moderate","livecrawl":"web"}"#; + + let output = handler + .execute("call_searxng", "web_search", arguments, ¶ms) + .await + .unwrap(); + + let request = captured.recv().await.expect("mock SearXNG should receive the request"); + assert_eq!(request.authorization, None, "SearXNG is keyless by default"); + assert_eq!(request.accept.as_deref(), Some("application/json")); + assert_eq!( + request.accept_encoding, None, + "core reqwest has no gzip support, so Accept-Encoding must never be sent" + ); + assert_eq!( + request.params, + serde_json::json!({ + "q": "rust async", + "format": "json", + "categories": "general,news", + "time_range": "week", + "language": "en-GB", + "safesearch": 1 + }), + "freshness uses time_range, safesearch is numeric, count/country/livecrawl are not sent" + ); + + assert_eq!(output.call_id, "call_searxng"); + assert_eq!( + output.output, + concat!( + r#"{"query":"rust async","queries":["rust async"],"#, + r#""results":{"web":["#, + r#"{"url":"https://example.com/rust","title":"Rust async guide","description":"A useful guide"},"#, + r#"{"url":"https://docs.example.org/tokio","title":"Tokio","description":"Runtime"}],"#, + r#""news":[{"url":"https://news.example.com/async-release","title":"Async release","#, + r#""description":"Released today","page_age":"2026-09-01T10:00:00"}]},"#, + r#""metadata":[{"provider":"searxng","query":"rust async"}]}"# + ) + ); + + let public = handler + .public_output(&call(arguments), &output, WebSearchCallStatus::Completed, ¶ms) + .expect("web_search_call public output"); + assert_eq!( + serde_json::to_value(&public).unwrap(), + serde_json::json!({ + "id": "ws_searxng", + "type": "web_search_call", + "status": "completed", + "action": { + "type": "search", + "query": "rust async", + "queries": ["rust async"], + "sources": [ + {"url": "https://example.com/rust", "title": "Rust async guide"}, + {"url": "https://docs.example.org/tokio", "title": "Tokio"}, + {"url": "https://news.example.com/async-release", "title": "Async release"} + ] + } + }) + ); +} + +#[tokio::test] +async fn searxng_handler_sends_bearer_token_when_a_key_is_configured() { + let (base_url, mut captured, _handle) = spawn_mock_json(StatusCode::OK, mixed_response()).await; + // A trailing slash on the configured endpoint must not double up. + let handler = searxng_handler(Some(&format!("{base_url}/")), Some(" proxy-token "), None); + + execute(&handler, r#"{"query":"q"}"#, &WebSearchToolParam::default()) + .await + .unwrap(); + + let request = captured.recv().await.unwrap(); + assert_eq!(request.authorization.as_deref(), Some("Bearer proxy-token")); +} + +#[tokio::test] +async fn searxng_handler_returns_empty_sections_without_error() { + for body in [ + serde_json::json!({"query": "nothing", "results": []}), + serde_json::json!({"query": "nothing", "results": null}), + serde_json::json!({}), + ] { + let (base_url, _captured, _handle) = spawn_mock_json(StatusCode::OK, body).await; + let handler = searxng_handler(Some(&base_url), None, None); + + let output = handler + .execute( + "call_searxng", + "web_search", + r#"{"query":"nothing"}"#, + &WebSearchToolParam::default(), + ) + .await + .unwrap(); + + let output_json: serde_json::Value = serde_json::from_str(&output.output).unwrap(); + assert_eq!(output_json["results"], serde_json::json!({"web": [], "news": []})); + assert_eq!( + output_json["metadata"], + serde_json::json!([{"provider": "searxng", "query": "nothing"}]) + ); + let public = handler + .public_output( + &call(r#"{"query":"nothing"}"#), + &output, + WebSearchCallStatus::Completed, + &WebSearchToolParam::default(), + ) + .unwrap(); + let OutputItem::WebSearchCall(item) = public else { + panic!("expected web_search_call"); + }; + let action = serde_json::to_value(&item).unwrap()["action"].clone(); + assert!( + action["sources"].as_array().is_none_or(Vec::is_empty), + "empty results must not invent sources: {action}" + ); + } +} + +#[tokio::test] +async fn searxng_handler_applies_domain_filters_client_side() { + let (base_url, mut captured, _handle) = spawn_mock_json(StatusCode::OK, mixed_response()).await; + let handler = searxng_handler(Some(&base_url), None, None); + + // Tool-level allowlist wins over the model's arguments and is enforced + // locally: SearXNG never sees a domain parameter. + let params = WebSearchToolParam { + filters: Some(WebSearchFilters { + allowed_domains: Some(vec!["Example.com".to_owned()]), + blocked_domains: None, + }), + ..WebSearchToolParam::default() + }; + let output = execute( + &handler, + r#"{"query":"rust async","include_domains":["example.org"]}"#, + ¶ms, + ) + .await + .unwrap(); + let request = captured.recv().await.unwrap(); + assert!( + request + .params + .as_object() + .unwrap() + .keys() + .all(|key| !key.contains("domain")), + "no domain parameter must reach SearXNG: {}", + request.params + ); + let output_json: serde_json::Value = serde_json::from_str(&output).unwrap(); + assert_eq!( + output_json["results"]["web"], + serde_json::json!([{"url": "https://example.com/rust", "title": "Rust async guide", "description": "A useful guide"}]) + ); + assert_eq!( + output_json["results"]["news"], + serde_json::json!([{"url": "https://news.example.com/async-release", "title": "Async release", + "description": "Released today", "page_age": "2026-09-01T10:00:00"}]) + ); + + // The model's blocklist applies on a label boundary to both sections. + let output = execute( + &handler, + r#"{"query":"rust async","exclude_domains":["news.example.com","example.org"]}"#, + &WebSearchToolParam::default(), + ) + .await + .unwrap(); + let output_json: serde_json::Value = serde_json::from_str(&output).unwrap(); + let urls = |section: &str| { + output_json["results"][section] + .as_array() + .unwrap() + .iter() + .map(|result| result["url"].as_str().unwrap().to_owned()) + .collect::>() + }; + assert_eq!(urls("web"), ["https://example.com/rust"]); + assert!(urls("news").is_empty()); +} + +#[tokio::test] +async fn searxng_handler_truncates_results_to_count_after_filtering() { + let results: Vec = (0..12) + .map(|index| { + let category = if index % 2 == 0 { "general" } else { "news" }; + let host = if index < 4 { "example.com" } else { "other.org" }; + serde_json::json!({"url": format!("https://{host}/{index}"), "title": format!("r{index}"), "category": category}) + }) + .collect(); + let (base_url, mut captured, _handle) = + spawn_mock_json(StatusCode::OK, serde_json::json!({"results": results})).await; + let handler = searxng_handler(Some(&base_url), None, None); + + let output = execute( + &handler, + r#"{"query":"q","count":1,"include_domains":["example.com"]}"#, + &WebSearchToolParam::default(), + ) + .await + .unwrap(); + let request = captured.recv().await.unwrap(); + assert!(request.params.get("count").is_none(), "SearXNG has no count parameter"); + let output_json: serde_json::Value = serde_json::from_str(&output).unwrap(); + assert_eq!( + output_json["results"], + serde_json::json!({ + "web": [{"url": "https://example.com/0", "title": "r0"}], + "news": [{"url": "https://example.com/1", "title": "r1"}] + }), + "filtering runs before truncation so the allowlist cannot starve a section" + ); + + // `search_context_size` supplies the default ceiling (`low` = 3) when the model omits `count`. + let params = WebSearchToolParam { + search_context_size: Some(WebSearchContextSize::Low), + ..WebSearchToolParam::default() + }; + let output = execute(&handler, r#"{"query":"q"}"#, ¶ms).await.unwrap(); + let output_json: serde_json::Value = serde_json::from_str(&output).unwrap(); + assert_eq!(output_json["results"]["web"].as_array().unwrap().len(), 3); + assert_eq!(output_json["results"]["news"].as_array().unwrap().len(), 3); + + // Without either, every hit the instance returned is passed on. + let output = execute(&handler, r#"{"query":"q"}"#, &WebSearchToolParam::default()) + .await + .unwrap(); + let output_json: serde_json::Value = serde_json::from_str(&output).unwrap(); + assert_eq!(output_json["results"]["web"].as_array().unwrap().len(), 6); + assert_eq!(output_json["results"]["news"].as_array().unwrap().len(), 6); +} + +#[tokio::test] +async fn searxng_handler_reports_disabled_json_format_on_403() { + let (base_url, mut captured, _handle) = spawn_mock_searxng( + StatusCode::FORBIDDEN, + Vec::new(), + MockBody::Raw { + content_type: "text/html; charset=utf-8", + body: "403 Forbidden", + }, + ) + .await; + let handler = searxng_handler(Some(&base_url), Some("proxy-token"), None); + + let error = execute(&handler, r#"{"query":"q"}"#, &WebSearchToolParam::default()) + .await + .unwrap_err(); + assert_eq!( + error, + "execution failed: SearXNG refused the request (403 Forbidden); enable the JSON format with \ + `search.formats: [html, json]` in settings.yml, or check SEARXNG_API_KEY if the instance is behind an \ + authenticating proxy" + ); + assert!(!error.contains("proxy-token")); + assert_eq!( + captured.recv().await.unwrap().authorization.as_deref(), + Some("Bearer proxy-token") + ); +} + +#[tokio::test] +async fn searxng_handler_reports_rejected_credential_without_leaking_it() { + let (base_url, _captured, _handle) = spawn_mock_json( + StatusCode::UNAUTHORIZED, + serde_json::json!({"error": "token proxy-token is not valid"}), + ) + .await; + let handler = searxng_handler(Some(&base_url), Some("proxy-token"), None); + + let error = execute(&handler, r#"{"query":"q"}"#, &WebSearchToolParam::default()) + .await + .unwrap_err(); + assert_eq!( + error, + "execution failed: SearXNG rejected the credential (401 Unauthorized); check SEARXNG_API_KEY" + ); + assert!(!error.contains("proxy-token"), "upstream body must not be echoed"); +} + +#[tokio::test] +async fn searxng_handler_surfaces_limiter_blocks_without_retrying() { + let (base_url, mut captured, _handle) = spawn_mock_searxng( + StatusCode::TOO_MANY_REQUESTS, + vec![("retry-after", "30")], + MockBody::Raw { + content_type: "text/plain", + body: "IP is on BLOCKLIST - HTTP header Accept-Encoding did not contain gzip nor deflate", + }, + ) + .await; + let handler = searxng_handler(Some(&base_url), None, None); + + let error = execute(&handler, r#"{"query":"q"}"#, &WebSearchToolParam::default()) + .await + .unwrap_err(); + assert_eq!( + error, + "execution failed: SearXNG rate limited the request (429 Too Many Requests); the gateway does not retry; \ + retry after 30. If the instance runs with `server.limiter: true`, its bot detection blocks the gateway \ + (which cannot send Accept-Encoding: gzip): add the gateway address to `botdetection.ip_lists.pass_ip` in \ + limiter.toml or disable the limiter" + ); + captured.recv().await.expect("one request"); + assert!( + captured.try_recv().is_err(), + "a rate-limited request must not be retried" + ); + + // Without Retry-After the message omits the hint but keeps the limiter guidance. + let (base_url, _captured, _handle) = spawn_mock_json(StatusCode::TOO_MANY_REQUESTS, serde_json::json!({})).await; + let error = execute( + &searxng_handler(Some(&base_url), None, None), + r#"{"query":"q"}"#, + &WebSearchToolParam::default(), + ) + .await + .unwrap_err(); + assert!( + error.starts_with( + "execution failed: SearXNG rate limited the request (429 Too Many Requests); the gateway does not retry. If" + ), + "{error}" + ); +} + +#[tokio::test] +async fn searxng_handler_reports_html_body_as_misconfigured_endpoint() { + let (base_url, _captured, _handle) = spawn_mock_searxng( + StatusCode::OK, + Vec::new(), + MockBody::Raw { + content_type: "text/html; charset=utf-8", + body: "SearXNG web UI", + }, + ) + .await; + let handler = searxng_handler(Some(&base_url), None, None); + + let error = execute(&handler, r#"{"query":"q"}"#, &WebSearchToolParam::default()) + .await + .unwrap_err(); + assert!( + error.starts_with("execution failed: SearXNG returned a non-JSON response ("), + "{error}" + ); + assert!( + error.ends_with( + "); confirm base_url points at a SearXNG instance and that `search.formats` in settings.yml includes `json`" + ), + "{error}" + ); +} + +#[tokio::test] +async fn searxng_handler_reports_parameter_errors_and_other_failures_with_status() { + let (base_url, _captured, _handle) = spawn_mock_json( + StatusCode::BAD_REQUEST, + serde_json::json!({"error": "Invalid value \"xx-XXX\" for parameter language"}), + ) + .await; + let error = execute( + &searxng_handler(Some(&base_url), None, None), + r#"{"query":"q"}"#, + &WebSearchToolParam::default(), + ) + .await + .unwrap_err(); + assert_eq!( + error, + r#"execution failed: SearXNG rejected a search parameter (400 Bad Request): {"error":"Invalid value \"xx-XXX\" for parameter language"}"# + ); + + let (base_url, _captured, _handle) = spawn_mock_searxng( + StatusCode::BAD_GATEWAY, + Vec::new(), + MockBody::Raw { + content_type: "text/plain", + body: "upstream unavailable", + }, + ) + .await; + let error = execute( + &searxng_handler(Some(&base_url), None, None), + r#"{"query":"q"}"#, + &WebSearchToolParam::default(), + ) + .await + .unwrap_err(); + assert_eq!( + error, + "execution failed: SearXNG returned 502 Bad Gateway: upstream unavailable" + ); +} + +#[tokio::test] +async fn searxng_handler_fails_without_base_url_naming_the_setting() { + for base_url in [None, Some(""), Some(" ")] { + let handler = searxng_handler(base_url, None, None); + let error = execute(&handler, r#"{"query":"q"}"#, &WebSearchToolParam::default()) + .await + .unwrap_err(); + assert_eq!(error, format!("invalid tool config: {SEARXNG_BASE_URL_HINT}")); + } +} + +/// Mock that records the peak number of in-flight requests. +async fn spawn_concurrency_tracking_searxng() -> (String, Arc, tokio::task::JoinHandle<()>) { + let active = Arc::new(AtomicUsize::new(0)); + let max_active = Arc::new(AtomicUsize::new(0)); + let app = Router::new() + .route( + SEARXNG_SEARCH_PATH, + get( + |State((active, max_active)): State<(Arc, Arc)>, uri: Uri| async move { + let now = active.fetch_add(1, Ordering::SeqCst) + 1; + max_active.fetch_max(now, Ordering::SeqCst); + tokio::time::sleep(Duration::from_millis(30)).await; + active.fetch_sub(1, Ordering::SeqCst); + let query = support::query_params_as_json(&uri)["q"] + .as_str() + .unwrap_or("unknown") + .to_owned(); + Json(serde_json::json!({ + "results": [{"url": format!("https://example.com/{}", query.replace(' ', "-")), "title": query}] + })) + }, + ), + ) + .with_state((active, Arc::clone(&max_active))); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let handle = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + (format!("http://{addr}"), max_active, handle) +} + +#[tokio::test] +async fn searxng_handler_inherits_gateway_concurrency_by_default() { + let (base_url, max_active, _handle) = spawn_concurrency_tracking_searxng().await; + let handler = searxng_handler(Some(&base_url), None, None); + + let output = execute( + &handler, + r#"{"queries":["one","two","three","four","five"]}"#, + &WebSearchToolParam::default(), + ) + .await + .unwrap(); + + let output_json: serde_json::Value = serde_json::from_str(&output).unwrap(); + assert_eq!(output_json["results"]["web"].as_array().unwrap().len(), 5); + assert_eq!(output_json["metadata"].as_array().unwrap().len(), 5); + let peak = max_active.load(Ordering::SeqCst); + assert!( + (2..=5).contains(&peak), + "peak concurrency {peak} should reflect the inherited gateway limit of 5" + ); +} + +#[tokio::test] +async fn searxng_handler_honors_a_lowered_concurrency_override() { + let (base_url, max_active, _handle) = spawn_concurrency_tracking_searxng().await; + let handler = searxng_handler(Some(&base_url), None, NonZeroUsize::new(1)); + + execute( + &handler, + r#"{"queries":["one","two","three","four","five"]}"#, + &WebSearchToolParam::default(), + ) + .await + .unwrap(); + + assert_eq!( + max_active.load(Ordering::SeqCst), + 1, + "the operator override serializes batched queries" + ); +} diff --git a/crates/agentic-server/src/config_file.rs b/crates/agentic-server/src/config_file.rs index 743dfc99..2580e913 100644 --- a/crates/agentic-server/src/config_file.rs +++ b/crates/agentic-server/src/config_file.rs @@ -12,13 +12,14 @@ use serde::{Deserialize, Serialize}; #[derive(Debug, Default, Deserialize, Serialize)] #[serde(default, deny_unknown_fields)] pub(crate) struct WebSearchFileConfig { - /// Search backend (`you` or `brave`); unset selects You.com. + /// Search backend (`you`, `brave`, or `searxng`); unset selects You.com. #[serde(skip_serializing_if = "Option::is_none")] pub provider: Option, #[serde(skip_serializing_if = "Option::is_none")] pub base_url: Option, /// Environment variable holding the provider's API key; unset uses the - /// provider's conventional variable (`YOU_API_KEY`, `BRAVE_API_KEY`). + /// provider's conventional variable (`YOU_API_KEY`, `BRAVE_API_KEY`, + /// `SEARXNG_API_KEY`); SearXNG needs no key unless a proxy demands one. #[serde(skip_serializing_if = "Option::is_none")] pub api_key_env: Option, /// Ceiling on concurrent provider requests within one batched search. @@ -608,6 +609,20 @@ mod tests { assert!(rendered.contains("provider = \"brave\"")); assert!(rendered.contains("max_concurrent_queries = 2")); + fs::write( + home.path().join("config.toml"), + "[web_search]\nprovider = \"searxng\"\nbase_url = \"http://searxng:8080\"\n", + ) + .expect("write config"); + let config = FileConfig::load(home.path()) + .expect("load config") + .expect("existing config"); + assert_eq!(config.web_search.provider, Some(WebSearchProviderKind::Searxng)); + assert_eq!(config.web_search.base_url.as_deref(), Some("http://searxng:8080")); + assert_eq!(config.web_search.api_key_env, None); + let rendered = toml::to_string(&config).expect("serialize config"); + assert!(rendered.contains("provider = \"searxng\"")); + fs::write(home.path().join("config.toml"), "[web_search]\nprovider = \"bing\"\n").expect("write config"); let error = FileConfig::load(home.path()).expect_err("unknown provider must fail"); assert!(error.to_string().contains("provider"), "{error}"); diff --git a/crates/agentic-server/src/web_search_config.rs b/crates/agentic-server/src/web_search_config.rs index 9c7ec766..9047d318 100644 --- a/crates/agentic-server/src/web_search_config.rs +++ b/crates/agentic-server/src/web_search_config.rs @@ -4,10 +4,11 @@ use std::num::NonZeroUsize; use agentic_core::config::{WebSearchProviderConfig, WebSearchProviderKind}; use agentic_core::error::Error; +use agentic_core::tool::SEARXNG_BASE_URL_HINT; use crate::config_file::WebSearchFileConfig; -/// Environment override for the `web_search` backend (`you` or `brave`). +/// Environment override for the `web_search` backend (`you`, `brave`, or `searxng`). const WEB_SEARCH_PROVIDER_ENV: &str = "AGENTIC_WEB_SEARCH_PROVIDER"; /// Provider-neutral environment override for the `web_search` endpoint. const WEB_SEARCH_BASE_URL_ENV: &str = "AGENTIC_WEB_SEARCH_BASE_URL"; @@ -25,6 +26,9 @@ const WEB_SEARCH_MAX_CONCURRENT_QUERIES_ENV: &str = "AGENTIC_WEB_SEARCH_MAX_CONC /// `AGENTIC_WEB_SEARCH_BASE_URL`, then `YOU_API_BASE_URL` (You.com only), then /// the file, then the provider default. `max_concurrent_queries` is left unset /// so the provider's own default applies. +/// +/// SearXNG has no default endpoint, so selecting it without a usable base URL +/// is rejected here rather than on the first `web_search` call. pub(crate) fn resolve_web_search_config( file: &WebSearchFileConfig, env: impl Fn(&str) -> Option, @@ -54,11 +58,29 @@ pub(crate) fn resolve_web_search_config( })?), None => file.max_concurrent_queries, }; + if provider == WebSearchProviderKind::Searxng { + validate_searxng_base_url(base_url.as_deref())?; + } Ok(WebSearchProviderConfig::new(api_key, base_url) .with_provider(provider) .with_max_concurrent_queries(max_concurrent_queries)) } +/// Rejects a missing or non-HTTP SearXNG endpoint with the same hint the +/// provider reports at execution time. +fn validate_searxng_base_url(base_url: Option<&str>) -> Result<(), Error> { + let value = base_url.map(str::trim).filter(|value| !value.is_empty()); + let Some(value) = value else { + return Err(Error::Config(SEARXNG_BASE_URL_HINT.to_owned())); + }; + match url::Url::parse(value) { + Ok(url) if matches!(url.scheme(), "http" | "https") && url.has_host() => Ok(()), + _ => Err(Error::Config(format!( + "SearXNG base URL {value:?} must be an absolute http(s) URL such as http://searxng:8080" + ))), + } +} + /// Seeds `[web_search]` in a generated configuration file from the current /// environment. Credentials stay unpinned so changing providers selects the /// corresponding default key variable; a malformed @@ -158,6 +180,67 @@ mod tests { assert_eq!(config.provider, WebSearchProviderKind::You); } + #[test] + fn web_search_config_selects_searxng_and_requires_its_endpoint() { + let config = resolve_web_search_config( + &WebSearchFileConfig::default(), + env_from(&[ + ("AGENTIC_WEB_SEARCH_PROVIDER", "searxng"), + ("AGENTIC_WEB_SEARCH_BASE_URL", "http://searxng:8080/"), + ("YOU_API_BASE_URL", "https://you.example"), + ("YOU_API_KEY", "you-secret"), + ]), + ) + .expect("resolve searxng"); + assert_eq!(config.provider, WebSearchProviderKind::Searxng); + assert_eq!(config.api_key, None, "SearXNG is keyless unless SEARXNG_API_KEY is set"); + assert_eq!(config.base_url.as_deref(), Some("http://searxng:8080/")); + assert_eq!(config.max_concurrent_queries, None); + + let file = WebSearchFileConfig { + provider: Some(WebSearchProviderKind::Searxng), + base_url: Some("https://search.internal/searxng".to_owned()), + ..WebSearchFileConfig::default() + }; + let config = resolve_web_search_config(&file, env_from(&[("SEARXNG_API_KEY", "proxy-token")])) + .expect("resolve searxng from file"); + assert_eq!(config.api_key.as_deref(), Some("proxy-token")); + assert_eq!(config.base_url.as_deref(), Some("https://search.internal/searxng")); + + let error = resolve_web_search_config( + &WebSearchFileConfig::default(), + env_from(&[("AGENTIC_WEB_SEARCH_PROVIDER", "searxng")]), + ) + .expect_err("missing endpoint"); + assert_eq!(error.to_string(), SEARXNG_BASE_URL_HINT); + + // A legacy You.com endpoint never stands in for the SearXNG one. + let error = resolve_web_search_config( + &WebSearchFileConfig::default(), + env_from(&[ + ("AGENTIC_WEB_SEARCH_PROVIDER", "searxng"), + ("YOU_API_BASE_URL", "https://you.example"), + ]), + ) + .expect_err("legacy endpoint ignored"); + assert_eq!(error.to_string(), SEARXNG_BASE_URL_HINT); + + for invalid in ["searxng:8080", "ftp://searxng:8080", "/searxng", "http://"] { + let error = resolve_web_search_config( + &WebSearchFileConfig::default(), + env_from(&[ + ("AGENTIC_WEB_SEARCH_PROVIDER", "searxng"), + ("AGENTIC_WEB_SEARCH_BASE_URL", invalid), + ]), + ) + .expect_err(invalid); + assert_eq!( + error.to_string(), + format!("SearXNG base URL {invalid:?} must be an absolute http(s) URL such as http://searxng:8080") + ); + } + } + #[test] fn web_search_config_applies_environment_precedence_for_endpoint_and_concurrency() { let file = WebSearchFileConfig { @@ -192,7 +275,7 @@ mod tests { .expect_err("unknown provider"); assert_eq!( error.to_string(), - "AGENTIC_WEB_SEARCH_PROVIDER: unknown web_search provider \"bing\"; expected one of: you, brave" + "AGENTIC_WEB_SEARCH_PROVIDER: unknown web_search provider \"bing\"; expected one of: you, brave, searxng" ); let error = resolve_web_search_config( @@ -231,15 +314,35 @@ mod tests { let generated = generated_web_search_file_config(env_from(&[("AGENTIC_WEB_SEARCH_PROVIDER", "bing")])); assert_eq!(generated.provider, Some(WebSearchProviderKind::You)); + + let generated = generated_web_search_file_config(env_from(&[ + ("AGENTIC_WEB_SEARCH_PROVIDER", "searxng"), + ("AGENTIC_WEB_SEARCH_BASE_URL", "http://searxng:8080"), + ])); + assert_eq!(generated.provider, Some(WebSearchProviderKind::Searxng)); + assert_eq!( + generated.base_url.as_deref(), + Some("http://searxng:8080"), + "the mandatory SearXNG endpoint is recorded so the file works on its own" + ); + assert_eq!(generated.api_key_env, None); } #[test] fn generated_web_search_config_can_switch_provider_without_pinning_credentials() { - for (initial, next, key) in [("you", "brave", "BRAVE_API_KEY"), ("brave", "you", "YOU_API_KEY")] { + for (initial, next, key) in [ + ("you", "brave", "BRAVE_API_KEY"), + ("brave", "you", "YOU_API_KEY"), + ("you", "searxng", "SEARXNG_API_KEY"), + ] { let generated = generated_web_search_file_config(env_from(&[("AGENTIC_WEB_SEARCH_PROVIDER", initial)])); let config = resolve_web_search_config( &generated, - env_from(&[("AGENTIC_WEB_SEARCH_PROVIDER", next), (key, "selected-secret")]), + env_from(&[ + ("AGENTIC_WEB_SEARCH_PROVIDER", next), + ("AGENTIC_WEB_SEARCH_BASE_URL", "http://search.example"), + (key, "selected-secret"), + ]), ) .expect("resolve switched provider"); assert_eq!(config.api_key.as_deref(), Some("selected-secret")); diff --git a/docs/deploying/README.md b/docs/deploying/README.md index 5b26fecf..42df2937 100644 --- a/docs/deploying/README.md +++ b/docs/deploying/README.md @@ -429,6 +429,20 @@ kubectl create secret generic agentic-api-secrets \ --from-literal=brave-api-key="$BRAVE_API_KEY" ``` +To keep search inside the cluster, point the gateway at a self-hosted +[SearXNG](https://docs.searxng.org/) Service instead. No Secret is needed; the base URL is +mandatory, and the instance must enable the JSON format (`search.formats: [html, json]` in +its `settings.yml`). If the instance runs with `server.limiter: true`, add the gateway's +Pod or Service CIDR to `botdetection.ip_lists.pass_ip` in its `limiter.toml`, because the +gateway never sends `Accept-Encoding: gzip` and would otherwise be rejected with `429`: + +```yaml + - name: AGENTIC_WEB_SEARCH_PROVIDER + value: searxng + - name: AGENTIC_WEB_SEARCH_BASE_URL + value: http://searxng.agentic-api.svc.cluster.local:8080 +``` + Do not commit API keys to the manifest or source tree. ## Optional: deploy with llm-d diff --git a/docs/deploying/kubernetes.md b/docs/deploying/kubernetes.md index 8d25e983..73698e8e 100644 --- a/docs/deploying/kubernetes.md +++ b/docs/deploying/kubernetes.md @@ -421,7 +421,12 @@ default You.com provider it is enabled when `YOU_API_KEY` and `YOU_API_BASE_URL` Secret and use the current You.com Search API base URL, `https://ydc-index.io`. To use Brave Search instead, set `AGENTIC_WEB_SEARCH_PROVIDER=brave` in the ConfigMap and store `BRAVE_API_KEY` in the Secret; no base URL is needed. The Brave free plan is rate limited to roughly one request per second, so the gateway runs batched queries serially by -default; raise `AGENTIC_WEB_SEARCH_MAX_CONCURRENT_QUERIES` only on a paid plan. +default; raise `AGENTIC_WEB_SEARCH_MAX_CONCURRENT_QUERIES` only on a paid plan. For a fully in-cluster deployment, set +`AGENTIC_WEB_SEARCH_PROVIDER=searxng` and `AGENTIC_WEB_SEARCH_BASE_URL` to the in-cluster URL of a self-hosted SearXNG +instance; no Secret is needed, and the server refuses to start if the base URL is missing. The SearXNG instance must +enable the JSON format (`search.formats: [html, json]` in `settings.yml`), and if its `server.limiter` is on, the +gateway's Pod CIDR must be listed in `botdetection.ip_lists.pass_ip` in `limiter.toml`, because the gateway never +sends `Accept-Encoding: gzip` and the limiter otherwise rejects it. Create the Secret from a protected environment file so the key does not enter shell history or process arguments: @@ -436,7 +441,8 @@ kubectl --namespace agentic-api create secret generic agentic-api-web-search \ The file contains one line, `YOU_API_KEY=...` (or `BRAVE_API_KEY=...` for Brave Search). Remove it securely after creating the Secret. Patch the environment in the production overlay (for Brave, replace the `YOU_API_BASE_URL` -operation with `path: /data/AGENTIC_WEB_SEARCH_PROVIDER`, `value: brave`): +operation with `path: /data/AGENTIC_WEB_SEARCH_PROVIDER`, `value: brave`; for SearXNG, skip the Secret and set +`AGENTIC_WEB_SEARCH_PROVIDER` to `searxng` plus `AGENTIC_WEB_SEARCH_BASE_URL` to the instance URL): ```yaml patches: @@ -494,4 +500,6 @@ Remove the `Authorization` header when inbound OIDC validation is disabled. A to only the response status. A `403 Forbidden` from You.com means the external search request was rejected; confirm the documented base URL and refresh the Secret, restart the Deployment, and test again without printing the key. A failed `web_search_call` naming `BRAVE_API_KEY` means Brave rejected the key; one reporting `429` means the Brave plan's rate -limit was hit, and the gateway does not retry it. +limit was hit, and the gateway does not retry it. With SearXNG, a `403` means the instance has not enabled the JSON +format, and a `429` means its limiter blocked the gateway; the failure message names the `settings.yml` or +`limiter.toml` change to make. From 7155f09add3459431a928fc9ff3f0141fc41da68 Mon Sep 17 00:00:00 2001 From: Zheng Lu Date: Fri, 18 Sep 2026 01:23:32 +0100 Subject: [PATCH 2/2] fix(tool): harden SearXNG endpoint validation and clarify docs after review Signed-off-by: Zheng Lu --- CHANGELOG.md | 13 ++- README.md | 11 +- crates/agentic-server-core/src/tool/mod.rs | 2 +- .../src/tool/web_search/searxng.rs | 110 +++++++++++++++++- .../tests/web_search_searxng_test.rs | 62 ++++++++++ .../agentic-server/src/web_search_config.rs | 37 +++--- docs/deploying/README.md | 16 ++- docs/deploying/kubernetes.md | 6 +- 8 files changed, 220 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ca25e99..746c8112 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,9 @@ All notable changes to Agentic API are documented here. - Added SearXNG as a selectable backend for the gateway-owned `web_search` tool (#326, Phase 3 of #291). Select it with `AGENTIC_WEB_SEARCH_PROVIDER=searxng` or `[web_search] provider = "searxng"` and point - `AGENTIC_WEB_SEARCH_BASE_URL` or `[web_search] base_url` at a self-hosted instance; the endpoint is mandatory and - the server refuses to start without it. No API key is needed; `SEARXNG_API_KEY` (or the variable named by + `AGENTIC_WEB_SEARCH_BASE_URL` or `[web_search] base_url` at a self-hosted instance; the endpoint is mandatory + (an absolute `http(s)` URL without a query or fragment, sub-path mounts allowed) and the server refuses to start + without it. No API key is needed; `SEARXNG_API_KEY` (or the variable named by `api_key_env`) is sent as a `Bearer` token only when set. Web and news results come from one `format=json&categories=general,news` request per query, split by category. The gateway adapts the shared tool contract: `allowed_domains` / `blocked_domains` and the model's `include_domains` / `exclude_domains` are enforced @@ -40,6 +41,14 @@ All notable changes to Agentic API are documented here. ### Changed +- `WebSearchProviderKind` gains a `Searxng` variant (`"searxng"`) with no default endpoint, `SEARXNG_API_KEY` as + its conventional key variable, and no provider concurrency ceiling. `WebSearchProviderKind::ALL` grows from + `[Self; 2]` to `[Self; 3]` (it enumerates every selectable provider and will grow again with each one); iterating + it is unaffected, but code that destructured or annotated the fixed length must be updated. + `agentic_core::tool::SEARXNG_BASE_URL_HINT` and `validate_searxng_base_url` carry the operator-facing rules for + the mandatory endpoint (absolute `http(s)` URL with a host and no query or fragment). The shared + `null_as_default` and `read_response_limited` helpers moved from `web_search/mod.rs` to `web_search/provider.rs` + (crate-private, re-exported unchanged). - Modeled the Codex model catalog and the upstream model listing as typed Rust structs instead of untyped JSON, and reported an undecodable upstream `/v1/models` payload as `502` rather than serving it as an empty catalog (#252). diff --git a/README.md b/README.md index 96744fba..053568f4 100644 --- a/README.md +++ b/README.md @@ -193,7 +193,7 @@ AGENTIC_WEB_SEARCH_PROVIDER=brave BRAVE_API_KEY= \ cargo run -p agentic-server -- --llm-api-base http://0.0.0.0:5050 ``` -Running fully on-premise? Point the gateway at a self-hosted [SearXNG](https://docs.searxng.org/) instance instead; +Prefer a self-hosted backend? Point the gateway at your own [SearXNG](https://docs.searxng.org/) instance instead; no API key is needed, only its URL: ```bash @@ -347,9 +347,12 @@ api_key_env = "BRAVE_API_KEY" ``` **SearXNG** (`provider = "searxng"`) runs against a [self-hosted SearXNG](https://docs.searxng.org/admin/installation.html) -instance, so prompts and search queries never leave your network and no API key is needed. The endpoint is -mandatory: the server refuses to start when `searxng` is selected without `AGENTIC_WEB_SEARCH_BASE_URL` or -`[web_search] base_url`. Two instance settings matter: +instance: the gateway talks only to your instance, no API key is needed, and no search vendor sees your +deployment. Note that SearXNG itself forwards each query to the engines enabled in its `settings.yml`, so for a +fully air-gapped setup restrict it to internal or offline engines. The endpoint is mandatory: the server refuses to +start when `searxng` is selected without `AGENTIC_WEB_SEARCH_BASE_URL` or `[web_search] base_url` (an absolute +`http(s)` URL without a query or fragment; a sub-path such as `http://host/searxng` is fine). Two instance settings +matter: - The JSON output format must be enabled: add `json` to `search.formats` in SearXNG's `settings.yml` (`formats: [html, json]`). Without it SearXNG answers `403`, which the failed `web_search_call` explains. diff --git a/crates/agentic-server-core/src/tool/mod.rs b/crates/agentic-server-core/src/tool/mod.rs index d0f8aca3..a904ab4d 100644 --- a/crates/agentic-server-core/src/tool/mod.rs +++ b/crates/agentic-server-core/src/tool/mod.rs @@ -28,4 +28,4 @@ pub use shell::ShellHandler; pub(crate) use tool_search::ToolSearchMetadata; pub use tool_search::{ToolSearchHandler, ToolSearchState}; pub use web_search::WebSearchHandler; -pub use web_search::searxng::SEARXNG_BASE_URL_HINT; +pub use web_search::searxng::{SEARXNG_BASE_URL_HINT, validate_searxng_base_url}; diff --git a/crates/agentic-server-core/src/tool/web_search/searxng.rs b/crates/agentic-server-core/src/tool/web_search/searxng.rs index dfe01e61..ab034914 100644 --- a/crates/agentic-server-core/src/tool/web_search/searxng.rs +++ b/crates/agentic-server-core/src/tool/web_search/searxng.rs @@ -39,6 +39,7 @@ use super::{ null_as_default, read_response_limited, }; use crate::config::WebSearchProviderKind; +use crate::error::Error; use crate::tool::handler::ToolError; use crate::types::tools::{WebSearchContextSize, WebSearchToolParam}; @@ -49,8 +50,55 @@ pub(crate) const SEARXNG_API_KEY: &str = WebSearchProviderKind::Searxng.default_ pub const SEARXNG_BASE_URL_HINT: &str = "SearXNG requires a base URL; set AGENTIC_WEB_SEARCH_BASE_URL or [web_search] base_url \ (for example http://searxng:8080)"; -const SEARCH_PATH: &str = "/search"; +const SEARCH_PATH: &str = "search"; const CATEGORIES: &str = "general,news"; + +/// Checks that a configured SearXNG endpoint can be addressed: an absolute +/// `http`/`https` URL with a host and no query or fragment, because the +/// provider appends `/search` to its path. A blank or missing value is +/// reported with [`SEARXNG_BASE_URL_HINT`]. Shared by the `agentic-server` +/// startup check and the provider so both reject the same inputs. +/// +/// # Errors +/// +/// Returns [`Error::Config`] with the operator-facing message when the value +/// is blank, not an absolute `http(s)` URL with a host, or carries a query or +/// fragment. +pub fn validate_searxng_base_url(value: Option<&str>) -> Result<(), Error> { + let value = value.map(str::trim).filter(|value| !value.is_empty()); + let Some(value) = value else { + return Err(Error::Config(SEARXNG_BASE_URL_HINT.to_owned())); + }; + parse_base_url(value).map(drop).map_err(Error::Config) +} + +/// Parses a non-blank endpoint, producing the operator-facing message on failure. +fn parse_base_url(value: &str) -> Result { + match url::Url::parse(value) { + Ok(url) if matches!(url.scheme(), "http" | "https") && url.has_host() => { + if url.query().is_some() || url.fragment().is_some() { + return Err(format!( + "SearXNG base URL {value:?} must not contain a query or fragment; the gateway appends /search to \ + its path" + )); + } + Ok(url) + } + _ => Err(format!( + "SearXNG base URL {value:?} must be an absolute http(s) URL such as http://searxng:8080" + )), + } +} + +/// Builds the `/search` endpoint under the configured base path, so a +/// sub-path mount such as `http://host/searxng` resolves to +/// `http://host/searxng/search`. +fn search_endpoint(base_url: &str) -> Result { + let mut url = parse_base_url(base_url).map_err(ToolError::Config)?; + let path = format!("{}/{SEARCH_PATH}", url.path().trim_end_matches('/')); + url.set_path(&path); + Ok(url) +} const NEWS_CATEGORY: &str = "news"; #[derive(Debug, Clone)] @@ -95,10 +143,11 @@ impl WebSearchProvider for SearxngSearchProvider { .base_url .as_deref() .ok_or_else(|| ToolError::Config(SEARXNG_BASE_URL_HINT.to_owned()))?; + let endpoint = search_endpoint(base_url)?; let request = SearxngSearchRequest::from_args_and_config(query, args, config)?; let mut builder = self .client - .get(format!("{base_url}{SEARCH_PATH}")) + .get(endpoint) .query(&request.query_params()) .header("Accept", "application/json"); if let Some(api_key) = &self.api_key { @@ -310,9 +359,10 @@ fn searxng_language(value: &str) -> Option { return None; } let region = subtags.find(|subtag| subtag.len() == 2 && subtag.bytes().all(|byte| byte.is_ascii_alphabetic())); - Some(region.map_or(primary.clone(), |region| { - format!("{primary}-{}", region.to_ascii_uppercase()) - })) + Some(match region { + Some(region) => format!("{primary}-{}", region.to_ascii_uppercase()), + None => primary, + }) } /// Maps the named `safesearch` levels onto SearXNG's `0` / `1` / `2`. @@ -444,6 +494,56 @@ mod tests { assert!(build_provider(None, None).base_url.is_none()); } + #[test] + fn search_endpoint_appends_search_under_the_base_path() { + assert_eq!( + search_endpoint("http://searxng:8080").unwrap().as_str(), + "http://searxng:8080/search" + ); + assert_eq!( + search_endpoint("https://search.internal/searxng").unwrap().as_str(), + "https://search.internal/searxng/search" + ); + assert_eq!( + search_endpoint("http://[::1]:8080/a/b").unwrap().as_str(), + "http://[::1]:8080/a/b/search" + ); + for invalid in [ + "searxng:8080", + "ftp://searxng", + "http://", + "http://host?x=y", + "http://host/#top", + ] { + let error = search_endpoint(invalid).expect_err(invalid).to_string(); + assert!(error.starts_with("invalid tool config: SearXNG base URL"), "{error}"); + } + } + + #[test] + fn validate_base_url_shares_the_provider_rules() { + assert!(validate_searxng_base_url(Some(" http://searxng:8080/ ")).is_ok()); + assert_eq!( + validate_searxng_base_url(None).unwrap_err().to_string(), + SEARXNG_BASE_URL_HINT + ); + assert_eq!( + validate_searxng_base_url(Some(" ")).unwrap_err().to_string(), + SEARXNG_BASE_URL_HINT + ); + assert_eq!( + validate_searxng_base_url(Some("http://host?x=y")) + .unwrap_err() + .to_string(), + "SearXNG base URL \"http://host?x=y\" must not contain a query or fragment; the gateway appends /search \ + to its path" + ); + assert_eq!( + validate_searxng_base_url(Some("/searxng")).unwrap_err().to_string(), + "SearXNG base URL \"/searxng\" must be an absolute http(s) URL such as http://searxng:8080" + ); + } + #[tokio::test] async fn search_without_base_url_names_the_setting() { let provider = build_provider(None, None); diff --git a/crates/agentic-server-core/tests/web_search_searxng_test.rs b/crates/agentic-server-core/tests/web_search_searxng_test.rs index 3146c8f5..6d6702ed 100644 --- a/crates/agentic-server-core/tests/web_search_searxng_test.rs +++ b/crates/agentic-server-core/tests/web_search_searxng_test.rs @@ -615,6 +615,68 @@ async fn searxng_handler_fails_without_base_url_naming_the_setting() { } } +#[tokio::test] +async fn searxng_handler_rejects_unaddressable_base_urls_without_sending() { + // Core callers bypass the server's startup check, so the provider applies + // the same rules before any request leaves the gateway. + let (base_url, mut captured, _handle) = spawn_mock_json(StatusCode::OK, mixed_response()).await; + for (suffix, needle) in [ + ("?format=json", "must not contain a query or fragment"), + ("/#search", "must not contain a query or fragment"), + ] { + let handler = searxng_handler(Some(&format!("{base_url}{suffix}")), None, None); + let error = execute(&handler, r#"{"query":"q"}"#, &WebSearchToolParam::default()) + .await + .unwrap_err(); + assert!(error.starts_with("invalid tool config: SearXNG base URL"), "{error}"); + assert!(error.contains(needle), "{error}"); + } + let error = execute( + &searxng_handler(Some("searxng:8080"), None, None), + r#"{"query":"q"}"#, + &WebSearchToolParam::default(), + ) + .await + .unwrap_err(); + assert!(error.contains("must be an absolute http(s) URL"), "{error}"); + assert!( + captured.try_recv().is_err(), + "no request may be sent for an invalid endpoint" + ); + + // A sub-path mount resolves to `{base}/search`. + let mounted = spawn_mounted_mock().await; + let handler = searxng_handler(Some(&format!("{}/searxng/", mounted.0)), None, None); + execute(&handler, r#"{"query":"q"}"#, &WebSearchToolParam::default()) + .await + .unwrap(); + assert_eq!(mounted.1.lock().await.as_deref(), Some("/searxng/search")); +} + +/// Mock that records the request path of the first `/searxng/search` hit. +async fn spawn_mounted_mock() -> ( + String, + Arc>>, + tokio::task::JoinHandle<()>, +) { + let seen = Arc::new(tokio::sync::Mutex::new(None)); + let app = Router::new() + .route( + "/searxng/search", + get( + |State(seen): State>>>, uri: Uri| async move { + *seen.lock().await = Some(uri.path().to_owned()); + Json(serde_json::json!({"results": []})) + }, + ), + ) + .with_state(Arc::clone(&seen)); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let handle = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + (format!("http://{addr}"), seen, handle) +} + /// Mock that records the peak number of in-flight requests. async fn spawn_concurrency_tracking_searxng() -> (String, Arc, tokio::task::JoinHandle<()>) { let active = Arc::new(AtomicUsize::new(0)); diff --git a/crates/agentic-server/src/web_search_config.rs b/crates/agentic-server/src/web_search_config.rs index 9047d318..f89106c3 100644 --- a/crates/agentic-server/src/web_search_config.rs +++ b/crates/agentic-server/src/web_search_config.rs @@ -4,7 +4,7 @@ use std::num::NonZeroUsize; use agentic_core::config::{WebSearchProviderConfig, WebSearchProviderKind}; use agentic_core::error::Error; -use agentic_core::tool::SEARXNG_BASE_URL_HINT; +use agentic_core::tool::validate_searxng_base_url; use crate::config_file::WebSearchFileConfig; @@ -28,7 +28,8 @@ const WEB_SEARCH_MAX_CONCURRENT_QUERIES_ENV: &str = "AGENTIC_WEB_SEARCH_MAX_CONC /// so the provider's own default applies. /// /// SearXNG has no default endpoint, so selecting it without a usable base URL -/// is rejected here rather than on the first `web_search` call. +/// is rejected here (via [`validate_searxng_base_url`]) rather than on the +/// first `web_search` call. pub(crate) fn resolve_web_search_config( file: &WebSearchFileConfig, env: impl Fn(&str) -> Option, @@ -59,6 +60,7 @@ pub(crate) fn resolve_web_search_config( None => file.max_concurrent_queries, }; if provider == WebSearchProviderKind::Searxng { + // SearXNG has no default endpoint; fail at startup with the provider's own message. validate_searxng_base_url(base_url.as_deref())?; } Ok(WebSearchProviderConfig::new(api_key, base_url) @@ -66,21 +68,6 @@ pub(crate) fn resolve_web_search_config( .with_max_concurrent_queries(max_concurrent_queries)) } -/// Rejects a missing or non-HTTP SearXNG endpoint with the same hint the -/// provider reports at execution time. -fn validate_searxng_base_url(base_url: Option<&str>) -> Result<(), Error> { - let value = base_url.map(str::trim).filter(|value| !value.is_empty()); - let Some(value) = value else { - return Err(Error::Config(SEARXNG_BASE_URL_HINT.to_owned())); - }; - match url::Url::parse(value) { - Ok(url) if matches!(url.scheme(), "http" | "https") && url.has_host() => Ok(()), - _ => Err(Error::Config(format!( - "SearXNG base URL {value:?} must be an absolute http(s) URL such as http://searxng:8080" - ))), - } -} - /// Seeds `[web_search]` in a generated configuration file from the current /// environment. Credentials stay unpinned so changing providers selects the /// corresponding default key variable; a malformed @@ -102,6 +89,8 @@ pub(crate) fn generated_web_search_file_config(env: impl Fn(&str) -> Option