From ed1cb97a6602efc71404ca0d2de7837fb32f81a2 Mon Sep 17 00:00:00 2001 From: blockmandev <194144408+blockmandev@users.noreply.github.com> Date: Thu, 22 Oct 2026 09:18:31 -0600 Subject: [PATCH 1/2] net: loopback never through a proxy; trust INCODE_CA_CERTS (refs #26) Every client is reqwest with its system proxy on, so HTTPS_PROXY, HTTP_PROXY, NO_PROXY and the Windows/macOS proxy settings already applied everywhere. But that matcher proxies loopback too unless NO_PROXY names it, so the memory engine, a local model server or a local MCP server went to a corporate proxy that cannot reach them, and a proxy re-signing TLS with a company CA had no way to be trusted. runtime::http::builder is now the base of every client: a client for this machine gets no proxy (client_for, no_redirect_client(url), the model API by its endpoint), and the PEM bundle INCODE_CA_CERTS (else SSL_CERT_FILE) names is trusted beside the system store; one that cannot be used is said and not added. Dev-only: rcgen and rustls (ring) for an end-to-end TLS test against a CA made at test time. --- Cargo.lock | 43 +++- Cargo.toml | 6 + README.md | 21 ++ src/agent/api.rs | 22 +- src/brief.rs | 6 +- src/ext_tools/fetch.rs | 4 +- src/mcp_oauth.rs | 6 +- src/mcp_oauth_tests.rs | 2 +- src/runtime/http.rs | 457 ++++++++++++++++++++++++++++++++++++-- src/toolbox/code_edits.rs | 2 +- src/toolbox/mcp_http.rs | 6 +- src/web.rs | 8 +- 12 files changed, 537 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cc4e9b4..f9e3b4b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1161,7 +1161,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2012,8 +2012,10 @@ dependencies = [ "pdf-extract", "pulldown-cmark", "quick-xml", + "rcgen", "regex", "reqwest", + "rustls", "rustyline", "serde", "serde_json", @@ -3009,6 +3011,16 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64 0.22.1", + "serde_core", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -3298,6 +3310,19 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a611d15b50743feb4c76b7d03edcb0e64f399c26961e4efe6975bc398be6aa3d" +[[package]] +name = "rcgen" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "yasna", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -3468,7 +3493,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3478,6 +3503,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ "once_cell", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -4057,7 +4083,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix 1.1.4", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4856,7 +4882,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5269,6 +5295,15 @@ version = "0.8.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e450f9b2ed1dff33c94c12589a87338689467b9c4f5d8a5710bd09a847d2c8a7" +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + [[package]] name = "yoke" version = "0.8.3" diff --git a/Cargo.toml b/Cargo.toml index c6ff426..0cb1e52 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -317,3 +317,9 @@ tempfile = "3" # `From>` impl needs the type. Already resolved to 1.5.0 as a # transitive dependency of reqwest, so this pins nothing new and is test-only. http = "1" +# The TLS server and the private certificate authority that prove a CA from +# `INCODE_CA_CERTS` is trusted (`runtime::http`). Generated at test time: no key +# is committed. Test-only; rustls and ring are already in Cargo.lock (as +# optional parts of reqwest's tree), rcgen is from the same project. +rcgen = "0.13" +rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } diff --git a/README.md b/README.md index a2b47af..9d529d5 100644 --- a/README.md +++ b/README.md @@ -1419,6 +1419,27 @@ there gives way), and the memory engine's files stay where they are. `INCODE_TRACE` (area `migrate`) says what moved. The old `INCORD_*` variables are not read any more: rename them. +### Proxies and certificates + +Behind a corporate proxy, every outbound request — the model API, web fetch and +search, MCP servers over HTTP and their OAuth, external tools and +`incode update`, the memory engine — follows `HTTPS_PROXY`, `HTTP_PROXY`, +`ALL_PROXY` and `NO_PROXY` (either case), and then the proxy set in Windows' +Internet Options or macOS' network settings. A request to **this machine** — +`localhost`, a `.localhost` name, `127.0.0.0/8` or `::1` — never goes through a +proxy, whatever `NO_PROXY` says, so a local memory engine, model server or MCP +server keeps working. Through a proxy, web fetch's check of where a name points +is made here, and the proxy resolves the name again itself: what the proxy can +reach is the proxy's to limit. + +When the proxy re-signs TLS with the company's own certificate authority, point +`INCODE_CA_CERTS` (or `SSL_CERT_FILE`) at its PEM file — one or more +certificates. They are trusted beside the system's own. A file that cannot be +read, or holds no certificate, is said on stderr at the first request and +nothing extra is trusted, so a connection that needed it fails with a +certificate error rather than going elsewhere. A coding agent CLI incode drives +(claude, codex, agy) makes its own requests and reads its own settings. + ## Security notes - **No subscription login is read or held.** The agent CLIs authenticate diff --git a/src/agent/api.rs b/src/agent/api.rs index be3c7bd..94bbd03 100644 --- a/src/agent/api.rs +++ b/src/agent/api.rs @@ -1205,10 +1205,14 @@ impl Transport for Http { .build() .map_err(|e| format!("no runtime for the request: {e}"))?; runtime.block_on(async move { - let client = reqwest::Client::builder() - .timeout(http.timeout) - .build() - .map_err(|e| format!("client: {e}"))?; + // A model server on this machine is reached directly, never + // through a proxy (`runtime::http`). + let client = crate::runtime::http::builder(crate::runtime::http::names_loopback( + &http.endpoint, + )) + .timeout(http.timeout) + .build() + .map_err(|e| format!("client: {e}"))?; let (response, retried) = http.post(&client, &payload).await?; let status = response.status(); let text = response @@ -1252,10 +1256,12 @@ impl Transport for Http { .build() .map_err(|e| format!("no runtime for the request: {e}"))?; runtime.block_on(async move { - let client = reqwest::Client::builder() - .timeout(http.timeout) - .build() - .map_err(|e| format!("client: {e}"))?; + let client = crate::runtime::http::builder(crate::runtime::http::names_loopback( + &http.endpoint, + )) + .timeout(http.timeout) + .build() + .map_err(|e| format!("client: {e}"))?; let (mut response, retried) = http.post(&client, &payload).await?; let status = response.status(); if !status.is_success() { diff --git a/src/brief.rs b/src/brief.rs index e05d5f3..a6dbb06 100644 --- a/src/brief.rs +++ b/src/brief.rs @@ -1613,13 +1613,13 @@ fn fields_not_stored(update: &BriefUpdate, saved: &ProjectBrief) -> Vec<&'static } lost } -use crate::runtime::http::{run_async, SHARED_CLIENT as HTTP_CLIENT}; +use crate::runtime::http::{client_for, run_async}; /// Fetch a URL reusing the pooled HTTP client and ambient/shared runtime, returning body and optional ETag. fn fetch_with_etag(url: &str) -> Result<(String, Option), String> { let url = url.to_string(); run_async(async move { - let response = HTTP_CLIENT + let response = client_for(&url) .get(&url) .timeout(FETCH_TIMEOUT) .send() @@ -1648,7 +1648,7 @@ fn post_with_etag(url: &str, body: String, etag: Option<&str>) -> Result Result { - let client = crate::runtime::http::no_redirect_client()?; let mut at = url.to_string(); for _ in 0..=MAX_REDIRECTS { crate::agent::catalogue::key_may_travel_to(&at) .map_err(|why| format!("refused to fetch it: {why}"))?; - let response = client + // Per hop: a redirect can lead off this machine, or onto it. + let response = crate::runtime::http::no_redirect_client(&at)? .get(&at) .header(reqwest::header::USER_AGENT, USER_AGENT) .header(reqwest::header::ACCEPT, accept) diff --git a/src/mcp_oauth.rs b/src/mcp_oauth.rs index fc94c4d..40c94b6 100644 --- a/src/mcp_oauth.rs +++ b/src/mcp_oauth.rs @@ -495,7 +495,7 @@ async fn read_bounded(mut response: reqwest::Response) -> Result /// Find the authorization server for the MCP server at `server`. async fn discover(server: String) -> Result { checked(&server, "the server's address")?; - let client = crate::runtime::http::no_redirect_client()?; + let client = crate::runtime::http::no_redirect_client(&server)?; let hello = json!({"jsonrpc": "2.0", "id": 0, "method": "initialize", "params": {"protocolVersion": crate::mcp_stdio::PROTOCOL_VERSION, "capabilities": {}, @@ -884,7 +884,7 @@ struct Client { /// Register incode as a public client (RFC 7591) for `redirect_uri`. async fn register(endpoint: String, redirect_uri: String) -> Result { checked(&endpoint, "its registration endpoint")?; - let client = crate::runtime::http::no_redirect_client()?; + let client = crate::runtime::http::no_redirect_client(&endpoint)?; let request = json!({ "client_name": "incode", "redirect_uris": [redirect_uri], @@ -988,7 +988,7 @@ async fn token_request( if let Some(resource) = &base.resource { form.push(("resource", resource.clone())); } - let client = crate::runtime::http::no_redirect_client()?; + let client = crate::runtime::http::no_redirect_client(&base.token_endpoint)?; let response = client .post(&base.token_endpoint) .header(ACCEPT, "application/json") diff --git a/src/mcp_oauth_tests.rs b/src/mcp_oauth_tests.rs index 8eecd81..4dc7f8a 100644 --- a/src/mcp_oauth_tests.rs +++ b/src/mcp_oauth_tests.rs @@ -689,7 +689,7 @@ fn browse(address: &str) { let address = address.to_string(); std::thread::spawn(move || { let location = crate::runtime::http::run_async(async move { - let client = crate::runtime::http::no_redirect_client()?; + let client = crate::runtime::http::no_redirect_client(&address)?; let answer = client .get(&address) .send() diff --git a/src/runtime/http.rs b/src/runtime/http.rs index 8260a3b..919d28e 100644 --- a/src/runtime/http.rs +++ b/src/runtime/http.rs @@ -3,17 +3,137 @@ //! Reuses a single multi-threaded Tokio runtime and `reqwest::Client` across //! Incord service requests (memory, brief, guards, owner, auditor, etc.) //! rather than building a new runtime, thread, and connection pool on every request. +//! +//! # Proxies and certificate authorities +//! +//! Every outbound client is built from [`builder`]: +//! +//! - **Proxy.** reqwest's own system proxy stands: `HTTPS_PROXY`, +//! `HTTP_PROXY`, `ALL_PROXY` and `NO_PROXY`, then the Windows or macOS +//! proxy settings. Its matcher sends a loopback address through the proxy +//! too unless `NO_PROXY` names it, which would put the local memory engine, +//! a local model server or a local MCP server behind a corporate proxy that +//! cannot reach them. So a client for THIS machine ([`is_loopback`]) is +//! built with no proxy at all, and every other keeps the system's. +//! - **CA.** The PEM bundle named by `INCODE_CA_CERTS` (else `SSL_CERT_FILE`) +//! is trusted beside the system's store. One that cannot be read or holds +//! no certificate is said on stderr and not added, so a TLS connection that +//! needed it fails rather than quietly trusting something else. +//! +//! Built here: the shared clients below, and the model API's +//! (`agent::api`) and web fetch/search's (`web`) own clients. use std::future::Future; use std::sync::LazyLock; use std::time::Duration; -pub static SHARED_CLIENT: LazyLock = LazyLock::new(|| { - reqwest::Client::builder() +/// Whether `url` names this machine: `localhost`, a `.localhost` name, or a +/// loopback address. Such a request never goes through a proxy. +#[must_use] +pub fn is_loopback(url: &reqwest::Url) -> bool { + let Some(host) = url.host_str() else { + return false; + }; + let host = host.trim_start_matches('[').trim_end_matches(']'); + match host.parse::() { + Ok(ip) => ip.is_loopback(), + Err(_) => { + let name = host.trim_end_matches('.').to_ascii_lowercase(); + name == "localhost" || name.ends_with(".localhost") + } + } +} + +/// Whether the address `url` names this machine; see [`is_loopback`]. A URL +/// that does not parse is not. +#[must_use] +pub fn names_loopback(url: &str) -> bool { + reqwest::Url::parse(url).is_ok_and(|url| is_loopback(&url)) +} + +/// The base every outbound client is built from: the operator's extra CA +/// certificates added, and, for a client that only talks to this machine +/// (`loopback`), no proxy. See the module docs. +pub fn builder(loopback: bool) -> reqwest::ClientBuilder { + let mut builder = reqwest::Client::builder(); + for certificate in EXTRA_ROOTS.iter() { + builder = builder.add_root_certificate(certificate.clone()); + } + if loopback { + builder = builder.no_proxy(); + } + builder +} + +/// The operator's extra certificate authorities, read once. +static EXTRA_ROOTS: LazyLock> = LazyLock::new(|| { + let (roots, warning) = roots_from(&|name| std::env::var_os(name)); + if let Some(warning) = warning { + eprintln!("incode: {warning}"); + } + roots +}); + +/// The certificates in the PEM bundle `INCODE_CA_CERTS` (else +/// `SSL_CERT_FILE`) names, as `get` reads the environment, and what to say +/// when it names one that cannot be used. +fn roots_from( + get: &dyn Fn(&str) -> Option, +) -> (Vec, Option) { + let Some((name, path)) = ["INCODE_CA_CERTS", "SSL_CERT_FILE"] + .into_iter() + .find_map(|name| { + get(name) + .filter(|value| !value.is_empty()) + .map(|value| (name, std::path::PathBuf::from(value))) + }) + else { + return (Vec::new(), None); + }; + let unused = |why: String| { + ( + Vec::new(), + Some(format!( + "{name}={}: {why}; no extra certificate authority is trusted", + path.display() + )), + ) + }; + let pem = match std::fs::read(&path) { + Ok(pem) => pem, + Err(e) => return unused(format!("could not be read ({e})")), + }; + match reqwest::Certificate::from_pem_bundle(&pem) { + Ok(roots) if !roots.is_empty() => (roots, None), + Ok(_) => unused("holds no PEM certificate".to_string()), + Err(e) => unused(format!("is not a PEM certificate bundle ({e})")), + } +} + +/// The client for incode's own services, with a 60 s bound on each request; +/// [`SHARED_DIRECT`] is its twin for this machine. Use [`client_for`]. +pub static SHARED_CLIENT: LazyLock = LazyLock::new(|| shared(false)); + +/// [`SHARED_CLIENT`] with no proxy, for a service on this machine. +static SHARED_DIRECT: LazyLock = LazyLock::new(|| shared(true)); + +fn shared(loopback: bool) -> reqwest::Client { + builder(loopback) .timeout(Duration::from_secs(60)) .build() .unwrap_or_default() -}); +} + +/// The shared client for a request to `url`: no proxy when it names this +/// machine. +#[must_use] +pub fn client_for(url: &str) -> &'static reqwest::Client { + if names_loopback(url) { + &SHARED_DIRECT + } else { + &SHARED_CLIENT + } +} /// How long a connection to a server a client named may take to open. pub const CONNECT_BOUND: Duration = Duration::from_secs(10); @@ -25,17 +145,29 @@ pub const CONNECT_BOUND: Duration = Duration::from_secs(10); /// /// `Err` when the client could not be built, which is said rather than /// replaced by a default client that WOULD follow redirects. -static NO_REDIRECT_CLIENT: LazyLock> = LazyLock::new(|| { - reqwest::Client::builder() +static NO_REDIRECT_CLIENT: LazyLock> = + LazyLock::new(|| no_redirect(false)); + +/// [`NO_REDIRECT_CLIENT`] with no proxy, for a server on this machine. +static NO_REDIRECT_DIRECT: LazyLock> = + LazyLock::new(|| no_redirect(true)); + +fn no_redirect(loopback: bool) -> Result { + builder(loopback) .redirect(reqwest::redirect::Policy::none()) .connect_timeout(CONNECT_BOUND) .build() .map_err(|e| format!("no HTTP client: {e}")) -}); +} -/// The shared client that follows no redirect; see [`NO_REDIRECT_CLIENT`]. -pub fn no_redirect_client() -> Result<&'static reqwest::Client, String> { - NO_REDIRECT_CLIENT.as_ref().map_err(Clone::clone) +/// The shared client that follows no redirect, for a request to `url`; see +/// [`NO_REDIRECT_CLIENT`]. No proxy when `url` names this machine. +pub fn no_redirect_client(url: &str) -> Result<&'static reqwest::Client, String> { + if names_loopback(url) { + NO_REDIRECT_DIRECT.as_ref().map_err(Clone::clone) + } else { + NO_REDIRECT_CLIENT.as_ref().map_err(Clone::clone) + } } pub static SHARED_RUNTIME: LazyLock = LazyLock::new(|| { @@ -71,16 +203,28 @@ where } } -/// Execute an HTTP request with `SHARED_CLIENT` and return the response text if successful. +/// Send the request `build` makes with the shared client its address calls +/// for ([`client_for`]): built first, so the address it names is known. +async fn send_shared(build: F) -> Result +where + F: FnOnce(&reqwest::Client) -> reqwest::RequestBuilder, +{ + let request = build(&SHARED_CLIENT) + .build() + .map_err(|e| format!("request failed: {}", error_chain(&e)))?; + client_for(request.url().as_str()) + .execute(request) + .await + .map_err(|e| format!("request failed: {}", error_chain(&e))) +} + +/// Execute an HTTP request with the shared client and return the response text if successful. pub fn request_text(build: F, service_name: &'static str) -> Result where F: FnOnce(&reqwest::Client) -> reqwest::RequestBuilder + Send + 'static, { run_async(async move { - let response = build(&SHARED_CLIENT) - .send() - .await - .map_err(|e| format!("request failed: {}", error_chain(&e)))?; + let response = send_shared(build).await?; let status = response.status(); if !status.is_success() { return Err(format!("the {service_name} answered {status}")); @@ -92,16 +236,13 @@ where }) } -/// Execute an HTTP request with `SHARED_CLIENT` and return `()` if successful. +/// Execute an HTTP request with the shared client and return `()` if successful. pub fn request_empty(build: F, service_name: &'static str) -> Result<(), String> where F: FnOnce(&reqwest::Client) -> reqwest::RequestBuilder + Send + 'static, { run_async(async move { - let response = build(&SHARED_CLIENT) - .send() - .await - .map_err(|e| format!("request failed: {}", error_chain(&e)))?; + let response = send_shared(build).await?; let status = response.status(); if !status.is_success() { return Err(format!("the {service_name} answered {status}")); @@ -129,6 +270,284 @@ pub(crate) fn error_chain(error: &dyn std::error::Error) -> String { #[cfg(test)] mod tests { use super::*; + use std::io::Write as _; + + #[test] + fn this_machine_is_named_by_localhost_and_loopback_addresses_only() { + for url in [ + "http://localhost:9180/x", + "http://LOCALHOST./", + "http://app.localhost/", + "http://127.0.0.1:8080/", + "http://127.4.5.6/", + "http://[::1]:11434/v1", + ] { + assert!(names_loopback(url), "{url}"); + } + for url in [ + "https://api.example.com/v1", + "http://localhost.example.com/", + "http://10.0.0.1/", + "http://[::2]/", + "not a url", + ] { + assert!(!names_loopback(url), "{url}"); + } + } + + /// A private certificate authority and a server certificate for + /// `localhost` it signed: the CA's PEM, the server's certificate and its + /// key (both DER). Made at test time; nothing secret is committed. + fn private_ca() -> (String, Vec, Vec) { + let ca_key = rcgen::KeyPair::generate().expect("ca key"); + let mut ca = rcgen::CertificateParams::new(Vec::::new()).expect("ca params"); + ca.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + ca.key_usages = vec![ + rcgen::KeyUsagePurpose::KeyCertSign, + rcgen::KeyUsagePurpose::CrlSign, + rcgen::KeyUsagePurpose::DigitalSignature, + ]; + ca.distinguished_name + .push(rcgen::DnType::CommonName, "incode test CA"); + let ca_cert = ca.self_signed(&ca_key).expect("ca"); + let key = rcgen::KeyPair::generate().expect("server key"); + let mut server = + rcgen::CertificateParams::new(vec!["localhost".to_string()]).expect("server params"); + server + .distinguished_name + .push(rcgen::DnType::CommonName, "localhost"); + server.extended_key_usages = vec![rcgen::ExtendedKeyUsagePurpose::ServerAuth]; + let cert = server.signed_by(&key, &ca_cert, &ca_key).expect("server"); + (ca_cert.pem(), cert.der().to_vec(), key.serialize_der()) + } + + fn env_of( + pairs: Vec<(&'static str, std::path::PathBuf)>, + ) -> impl Fn(&str) -> Option { + move |name| { + pairs + .iter() + .find(|(n, _)| *n == name) + .map(|(_, v)| v.clone().into_os_string()) + } + } + + /// **THE OPERATOR'S CA BUNDLE IS READ, AND ONE THAT CANNOT BE USED IS + /// SAID**, naming the variable and the file; `INCODE_CA_CERTS` comes + /// before `SSL_CERT_FILE`. + #[test] + fn a_ca_bundle_is_read_from_the_environment_and_a_bad_one_is_said() { + let dir = tempfile::tempdir().expect("dir"); + let (first, ..) = private_ca(); + let (second, ..) = private_ca(); + let bundle = dir.path().join("bundle.pem"); + std::fs::write(&bundle, format!("{first}\n{second}")).expect("write"); + let garbage = dir.path().join("garbage.pem"); + std::fs::write(&garbage, "not a certificate").expect("write"); + let missing = dir.path().join("missing.pem"); + + let (roots, warning) = roots_from(&env_of(Vec::new())); + assert!(roots.is_empty() && warning.is_none()); + + let (roots, warning) = roots_from(&env_of(vec![ + ("INCODE_CA_CERTS", bundle.clone()), + ("SSL_CERT_FILE", missing.clone()), + ])); + assert_eq!(roots.len(), 2, "{warning:?}"); + assert!(warning.is_none(), "{warning:?}"); + + let (roots, warning) = roots_from(&env_of(vec![("SSL_CERT_FILE", bundle)])); + assert_eq!((roots.len(), warning), (2, None)); + + for (path, why) in [ + (missing, "could not be read"), + (garbage, "holds no PEM certificate"), + ] { + let (roots, warning) = roots_from(&env_of(vec![("INCODE_CA_CERTS", path.clone())])); + assert!(roots.is_empty()); + let warning = warning.unwrap_or_else(|| panic!("{} was not said", path.display())); + assert!( + warning.starts_with("INCODE_CA_CERTS=") && warning.contains(why), + "{warning}" + ); + } + } + + /// Set only in the child the network tests start: what to ask for. + const NET_CHILD: &str = "INCODE_NET_CHILD_URL"; + + /// The child half of the network tests: asks for the URL it was given + /// through the shared client, then — so a proxy can be seen being used — + /// for a name that exists nowhere, and prints the first answer. Returns + /// at once, passing, unless started as the child. + #[test] + fn net_child() { + let Ok(url) = std::env::var(NET_CHILD) else { + return; + }; + let said = request_text(move |c| c.get(&url), "test service"); + let _ = request_text(|c| c.get("http://incode-proxy-probe.invalid/"), "probe"); + println!("NET-CHILD: {said:?}"); + std::process::exit(0); + } + + /// Run [`net_child`] asking for `url`, with `env` set and no proxy or CA + /// setting inherited: what it printed. + fn ask_from_a_child(url: &str, env: &[(String, String)]) -> String { + let mut child = std::process::Command::new(std::env::current_exe().expect("exe")); + child.args([ + "--exact", + "runtime::http::tests::net_child", + "--nocapture", + "--test-threads=1", + "-q", + ]); + for name in [ + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "all_proxy", + "no_proxy", + "INCODE_CA_CERTS", + "SSL_CERT_FILE", + ] { + child.env_remove(name); + } + child + .env(NET_CHILD, url) + .envs(env.iter().map(|(k, v)| (k, v))); + let out = child.output().expect("the child runs"); + assert!(out.status.success(), "{out:?}"); + String::from_utf8_lossy(&out.stdout).into_owned() + } + + /// A plain HTTP server on 127.0.0.1 answering `body` to each of `times` + /// requests. + fn http_server(body: &'static str, times: usize) -> String { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind"); + let port = listener.local_addr().expect("addr").port(); + std::thread::spawn(move || { + for stream in listener.incoming().take(times) { + let Ok(mut stream) = stream else { continue }; + read_head(&mut stream); + let _ = write!( + stream, + "HTTP/1.1 200 OK\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", + body.len() + ); + } + }); + format!("http://127.0.0.1:{port}/") + } + + /// Read a request up to the end of its headers. + fn read_head(stream: &mut impl std::io::Read) { + let mut seen = Vec::new(); + let mut buf = [0u8; 1024]; + while !seen.windows(4).any(|w| w == b"\r\n\r\n") { + match stream.read(&mut buf) { + Ok(0) | Err(_) => return, + Ok(n) => seen.extend_from_slice(&buf[..n]), + } + } + } + + /// **THIS MACHINE IS REACHED DIRECTLY; ANYWHERE ELSE THROUGH THE PROXY.** + /// With `HTTP_PROXY` naming a proxy and no `NO_PROXY`, a request to a + /// loopback service gets its answer straight from it and never reaches + /// the proxy, while a remote name is asked of the proxy. + #[test] + fn a_loopback_request_skips_the_proxy_and_a_remote_one_takes_it() { + let proxy = incord_sandbox::proxy::Proxy::start( + incord_sandbox::proxy::Allowlist::new(&[], false).expect("allowlist"), + None, + ) + .expect("proxy"); + let direct = http_server("direct", 1); + let said = ask_from_a_child(&direct, &proxy.client_env()); + assert!(said.contains(r#"NET-CHILD: Ok("direct")"#), "{said}"); + + // The proxy notes a decision once it has answered; give it a moment. + let deadline = std::time::Instant::now() + Duration::from_secs(5); + let mut asked: Vec; + loop { + asked = proxy.decisions().into_iter().map(|d| d.target).collect(); + if !asked.is_empty() || std::time::Instant::now() > deadline { + break; + } + std::thread::sleep(Duration::from_millis(50)); + } + assert!( + asked + .iter() + .any(|t| t.contains("incode-proxy-probe.invalid")), + "a remote request did not take the proxy: {asked:?}" + ); + assert!( + !asked.iter().any(|t| t.contains("127.0.0.1")), + "a loopback request went through the proxy: {asked:?}" + ); + } + + /// A TLS server for `localhost` on 127.0.0.1, answering `ok` to each of + /// `times` connections: its URL. + fn tls_server(cert: Vec, key: Vec, times: usize) -> String { + use rustls::pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer}; + let config = rustls::ServerConfig::builder_with_provider(std::sync::Arc::new( + rustls::crypto::ring::default_provider(), + )) + .with_safe_default_protocol_versions() + .expect("versions") + .with_no_client_auth() + .with_single_cert( + vec![CertificateDer::from(cert)], + PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(key)), + ) + .expect("server config"); + let config = std::sync::Arc::new(config); + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind"); + let port = listener.local_addr().expect("addr").port(); + std::thread::spawn(move || { + for stream in listener.incoming().take(times) { + let Ok(tcp) = stream else { continue }; + let Ok(conn) = rustls::ServerConnection::new(std::sync::Arc::clone(&config)) else { + continue; + }; + let mut tls = rustls::StreamOwned::new(conn, tcp); + read_head(&mut tls); + let _ = tls.write_all( + b"HTTP/1.1 200 OK\r\ncontent-length: 2\r\nconnection: close\r\n\r\nok", + ); + tls.conn.send_close_notify(); + let _ = tls.flush(); + } + }); + format!("https://localhost:{port}/") + } + + /// **A SERVER SIGNED BY THE OPERATOR'S OWN CA IS TRUSTED ONCE + /// `INCODE_CA_CERTS` NAMES IT**, and not without it: the same request, + /// through the same shared client, refused and then answered. + #[test] + fn a_server_signed_by_the_operators_ca_is_trusted_through_incode_ca_certs() { + let (ca_pem, cert, key) = private_ca(); + let dir = tempfile::tempdir().expect("dir"); + let bundle = dir.path().join("corporate-ca.pem"); + std::fs::write(&bundle, ca_pem).expect("write"); + let url = tls_server(cert, key, 2); + + let untrusted = ask_from_a_child(&url, &[]); + assert!(untrusted.contains("NET-CHILD: Err("), "{untrusted}"); + + let trusted = ask_from_a_child( + &url, + &[("INCODE_CA_CERTS".to_string(), bundle.display().to_string())], + ); + assert!(trusted.contains(r#"NET-CHILD: Ok("ok")"#), "{trusted}"); + } #[test] fn run_async_executes_from_synchronous_context() { diff --git a/src/toolbox/code_edits.rs b/src/toolbox/code_edits.rs index ca5b34d..ad5d593 100644 --- a/src/toolbox/code_edits.rs +++ b/src/toolbox/code_edits.rs @@ -125,7 +125,7 @@ impl EditLedger for EngineLedger { ); let body = body.clone(); let answered = crate::runtime::http::run_async(async move { - let response = match crate::runtime::http::SHARED_CLIENT + let response = match crate::runtime::http::client_for(&url) .post(&url) .timeout(CALL_BOUND) .json(&body) diff --git a/src/toolbox/mcp_http.rs b/src/toolbox/mcp_http.rs index aa38588..f18bb97 100644 --- a/src/toolbox/mcp_http.rs +++ b/src/toolbox/mcp_http.rs @@ -102,7 +102,7 @@ impl HttpLink { what: &str, ) -> Result { crate::agent::catalogue::key_may_travel_to(url)?; - crate::runtime::http::no_redirect_client()?; + crate::runtime::http::no_redirect_client(url)?; let mut map = HeaderMap::new(); for (index, (name, value)) in headers.iter().enumerate() { // Neither the name nor the value goes into the message: a value @@ -160,7 +160,7 @@ impl HttpLink { impl Link for HttpLink { fn send(&mut self, line: &str) -> Result<(), String> { - let client = crate::runtime::http::no_redirect_client()?; + let client = crate::runtime::http::no_redirect_client(&self.url)?; // A request has an id and must be answered; a notification has none. let id = serde_json::from_str::(line) .ok() @@ -242,7 +242,7 @@ impl Drop for HttpLink { let Some(session) = lock(&self.session).take() else { return; }; - let Ok(client) = crate::runtime::http::no_redirect_client() else { + let Ok(client) = crate::runtime::http::no_redirect_client(&self.url) else { return; }; let mut headers = self.headers.clone(); diff --git a/src/web.rs b/src/web.rs index 4d9777d..cf1984d 100644 --- a/src/web.rs +++ b/src/web.rs @@ -569,7 +569,7 @@ fn encode(value: &str) -> String { /// Ask the search provider, bounded in time and size, following no redirect. fn search_get(url: String, key: Option) -> Result { crate::runtime::http::run_async(async move { - let client = reqwest::Client::builder() + let client = crate::runtime::http::builder(false) .timeout(SEARCH_TIMEOUT) .redirect(reqwest::redirect::Policy::none()) .build() @@ -623,7 +623,11 @@ fn fetch_page(url: &str) -> Result<(String, String), String> { .build() .map_err(|e| format!("no runtime for the request: {e}"))?; runtime.block_on(async move { - let client = reqwest::Client::builder() + // The system's proxy applies (`runtime::http`). Through one, the + // PROXY resolves the name, so the pin below cannot hold there: the + // checks above still refuse a name that resolves inward here, and + // what the proxy itself may reach is the proxy's to bound. + let client = crate::runtime::http::builder(false) .timeout(TIMEOUT) // The checked address, used instead of resolving again. .resolve(&host, pinned) From 20e428d17a65aff66a1ec4b467024e5ae974ac73 Mon Sep 17 00:00:00 2001 From: blockmandev <194144408+blockmandev@users.noreply.github.com> Date: Thu, 22 Oct 2026 09:32:10 -0600 Subject: [PATCH 2/2] net: a direct client stays on this machine; OAuth metadata picks its own client Review of the previous commit: a client with no proxy followed any redirect, so a local service redirecting elsewhere sent the request to the other host past the proxy; and OAuth discovery reused the client chosen for the MCP server for metadata URLs on other hosts. builder(true) now follows a redirect only while it stays on this machine, and mcp_oauth::fetch_json chooses the client for the URL it fetches. --- src/mcp_oauth.rs | 13 ++++++---- src/runtime/http.rs | 62 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 69 insertions(+), 6 deletions(-) diff --git a/src/mcp_oauth.rs b/src/mcp_oauth.rs index 40c94b6..0618618 100644 --- a/src/mcp_oauth.rs +++ b/src/mcp_oauth.rs @@ -461,9 +461,11 @@ pub(crate) struct Discovered { pub(crate) scopes: Vec, } -/// A JSON document at `url`: `None` for an answer that is not a success. -async fn fetch_json(client: &reqwest::Client, url: &str) -> Result, String> { - let response = client +/// A JSON document at `url`: `None` for an answer that is not a success. The +/// client is chosen for `url` itself — metadata can live on another host than +/// the server, and only an address on this machine skips the proxy. +async fn fetch_json(url: &str) -> Result, String> { + let response = crate::runtime::http::no_redirect_client(url)? .get(url) .header(ACCEPT, "application/json") .timeout(HTTP_BOUND) @@ -495,6 +497,7 @@ async fn read_bounded(mut response: reqwest::Response) -> Result /// Find the authorization server for the MCP server at `server`. async fn discover(server: String) -> Result { checked(&server, "the server's address")?; + // For the server only: each metadata document picks its own (`fetch_json`). let client = crate::runtime::http::no_redirect_client(&server)?; let hello = json!({"jsonrpc": "2.0", "id": 0, "method": "initialize", "params": {"protocolVersion": crate::mcp_stdio::PROTOCOL_VERSION, @@ -530,7 +533,7 @@ async fn discover(server: String) -> Result { candidates.extend(resource_metadata_candidates(&server)?); let mut resource_metadata = None; for candidate in &candidates { - if let Some(text) = fetch_json(client, candidate).await? { + if let Some(text) = fetch_json(candidate).await? { resource_metadata = Some(parse_resource_metadata(&text, &server)?); break; } @@ -561,7 +564,7 @@ async fn discover(server: String) -> Result { } }; for candidate in as_metadata_candidates(&issuer)? { - if let Some(text) = fetch_json(client, &candidate).await? { + if let Some(text) = fetch_json(&candidate).await? { return Ok(Discovered { server: parse_server_metadata(&text, &issuer)?, resource, diff --git a/src/runtime/http.rs b/src/runtime/http.rs index 919d28e..5e0dace 100644 --- a/src/runtime/http.rs +++ b/src/runtime/http.rs @@ -54,17 +54,36 @@ pub fn names_loopback(url: &str) -> bool { /// The base every outbound client is built from: the operator's extra CA /// certificates added, and, for a client that only talks to this machine /// (`loopback`), no proxy. See the module docs. +/// +/// A client with no proxy follows a redirect only while it STAYS on this +/// machine: one that led elsewhere would be fetched without the proxy, past +/// whatever the proxy is there to enforce. A caller that follows none sets +/// its own policy over this one. pub fn builder(loopback: bool) -> reqwest::ClientBuilder { let mut builder = reqwest::Client::builder(); for certificate in EXTRA_ROOTS.iter() { builder = builder.add_root_certificate(certificate.clone()); } if loopback { - builder = builder.no_proxy(); + builder = builder + .no_proxy() + .redirect(reqwest::redirect::Policy::custom(|attempt| { + if !is_loopback(attempt.url()) { + attempt.error("a redirect from this machine to another is not followed") + } else if attempt.previous().len() >= MAX_LOCAL_REDIRECTS { + attempt.stop() + } else { + attempt.follow() + } + })); } builder } +/// How many redirects a client for this machine follows, as reqwest's own +/// default does. +const MAX_LOCAL_REDIRECTS: usize = 10; + /// The operator's extra certificate authorities, read once. static EXTRA_ROOTS: LazyLock> = LazyLock::new(|| { let (roots, warning) = roots_from(&|name| std::env::var_os(name)); @@ -295,6 +314,47 @@ mod tests { } } + /// **THE CLIENT IS CHOSEN FOR EACH ADDRESS**: one on this machine gets + /// the direct client, anything else the proxied one — so a caller that + /// reaches two hosts must ask for each (`mcp_oauth::fetch_json`). + #[test] + fn the_client_is_chosen_by_the_address_it_is_for() { + let local = "http://127.0.0.1:9180/x"; + let remote = "https://auth.example.com/.well-known/oauth-authorization-server"; + assert!(std::ptr::eq(client_for(local), &*SHARED_DIRECT)); + assert!(std::ptr::eq(client_for(remote), &*SHARED_CLIENT)); + let (Ok(direct), Ok(proxied)) = (no_redirect_client(local), no_redirect_client(remote)) + else { + panic!("no client"); + }; + assert!(!std::ptr::eq(direct, proxied)); + assert!(std::ptr::eq( + direct, + no_redirect_client("http://localhost/").expect("client") + )); + } + + /// **A CLIENT FOR THIS MACHINE DOES NOT FOLLOW A REDIRECT OFF IT**: it + /// has no proxy, so following would reach the other host past the proxy. + #[test] + fn a_redirect_from_this_machine_to_another_is_not_followed() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind"); + let port = listener.local_addr().expect("addr").port(); + std::thread::spawn(move || { + if let Some(Ok(mut stream)) = listener.incoming().next() { + read_head(&mut stream); + let _ = write!( + stream, + "HTTP/1.1 302 Found\r\nlocation: http://incode-redirect-probe.invalid/\r\n\ + content-length: 0\r\nconnection: close\r\n\r\n" + ); + } + }); + let url = format!("http://127.0.0.1:{port}/"); + let said = request_text(move |c| c.get(&url), "test service").expect_err("redirected"); + assert!(said.contains("is not followed"), "{said}"); + } + /// A private certificate authority and a server certificate for /// `localhost` it signed: the CA's PEM, the server's certificate and its /// key (both DER). Made at test time; nothing secret is committed.