From 5dea1ea7e2a5c6edb6d1be334d1bb9a0ab10734f Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Thu, 5 Nov 2026 13:48:45 +0800 Subject: [PATCH 1/4] Support stacked vendor patches in extconf Detect already-applied patches by finding the newest patch that reverse-applies cleanly, rather than reverse-checking each patch independently. The old check broke once a later patch touched the same file as an earlier one, causing extconf to re-apply the earlier patch and abort. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ext/wreq_rb/extconf.rb | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/ext/wreq_rb/extconf.rb b/ext/wreq_rb/extconf.rb index 1840477..2d5e092 100644 --- a/ext/wreq_rb/extconf.rb +++ b/ext/wreq_rb/extconf.rb @@ -8,9 +8,16 @@ wreq_dir = File.expand_path("../../vendor/wreq", __dir__) if File.directory?(patch_dir) - Dir.glob(File.join(patch_dir, "*.patch")).sort.each do |patch| - check = `cd #{wreq_dir} && git apply --check --reverse #{patch} 2>&1` - if $?.success? + patches = Dir.glob(File.join(patch_dir, "*.patch")).sort + # Patches stack in order and may touch the same files, so a patch only reverse-applies + # cleanly while no later patch is applied. The newest patch that reverse-applies marks + # itself and every earlier patch as already applied. + applied_count = (patches.rindex do |patch| + system("git", "apply", "--check", "--reverse", patch, chdir: wreq_dir, out: File::NULL, err: File::NULL) + end || -1) + 1 + + patches.each_with_index do |patch, index| + if index < applied_count puts "Patch already applied: #{File.basename(patch)}" else puts "Applying patch: #{File.basename(patch)}" From b888fcb8f886b961ea1da4b36e79b36474ef1371 Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Thu, 5 Nov 2026 13:48:45 +0800 Subject: [PATCH 2/4] Add preconnect support to vendored wreq Add Client::preconnect and RequestBuilder::preconnect, which establish a pooled connection (TCP, proxy tunnel, TLS) without sending a request. The connection descriptor is derived the same way as for a real request, so per-request proxy and emulation options yield a reusable pool key. The scheme/https_only check and total timeout normally applied by the middleware stack are replicated, since preconnect bypasses it. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- patches/0002-add-preconnect.patch | 266 ++++++++++++++++++++++++++++++ 1 file changed, 266 insertions(+) create mode 100644 patches/0002-add-preconnect.patch diff --git a/patches/0002-add-preconnect.patch b/patches/0002-add-preconnect.patch new file mode 100644 index 0000000..3666741 --- /dev/null +++ b/patches/0002-add-preconnect.patch @@ -0,0 +1,266 @@ +diff --git a/src/client.rs b/src/client.rs +index ccbdb0e9..d96bc245 100644 +--- a/src/client.rs ++++ b/src/client.rs +@@ -48,7 +48,9 @@ use self::{ + config::{ConfigService, ConfigServiceLayer}, + redirect::{FollowRedirect, FollowRedirectLayer}, + retry::RetryPolicy, +- timeout::{Timeout, TimeoutLayer, TimeoutOptions, body::TimeoutBody}, ++ timeout::{ ++ Timeout, TimeoutLayer, TimeoutOptions, body::TimeoutBody, fetch_timeout_options, ++ }, + transfer_size::{CountingBody, TransferSizeLayer, TransferSizeService}, + }, + request::{Request, RequestBuilder}, +@@ -60,6 +62,7 @@ use crate::cookie; + use crate::dns::hickory::HickoryDnsResolver; + use crate::{ + IntoUri, Method, Proxy, ++ config::RequestConfig, + conn::{ + BoxedConnectorLayer, BoxedTransportConnector, Conn, Unnameable, + connector::{Connector, ConnectorBuilder}, +@@ -68,6 +71,7 @@ use crate::{ + }, + dns::{DnsResolverWithOverrides, DynResolver, GaiResolver, IntoResolve, Resolve}, + error::{self, Error}, ++ ext::UriExt, + header::OrigHeaderMap, + http1::Http1Options, + http2::Http2Options, +@@ -154,7 +158,23 @@ type BoxedClientServiceLayer = BoxCloneSyncServiceLayer< + /// + /// [`Rc`]: std::rc::Rc + #[derive(Clone)] +-pub struct Client(Arc>); ++pub struct Client(Arc); ++ ++struct ClientRef { ++ service: Either, ++ preconnector: Preconnector, ++} ++ ++/// Direct handle to the connection pool used by [`Client::preconnect`]. ++/// ++/// Preconnecting bypasses the request middleware stack, so the checks that ++/// stack would apply before a connection is opened are replicated here. ++struct Preconnector { ++ http: HttpClient, ++ https_only: bool, ++ timer: Timer, ++ timeout: RequestConfig, ++} + + /// A [`ClientBuilder`] can be used to create a [`Client`] with custom configuration. + #[must_use] +@@ -430,7 +450,67 @@ impl Client { + let req = http::Request::::from(request); + Pending::Request { + uri: Some(req.uri().clone()), +- fut: Box::pin(Oneshot::new((*self.0).clone(), req)), ++ fut: Box::pin(Oneshot::new(self.0.service.clone(), req)), ++ } ++ } ++ ++ /// Establishes a connection to the given URI ahead of time, without sending a request. ++ /// ++ /// The TCP connection (plus proxy tunnel and TLS handshake, where applicable) is ++ /// completed and the connection is kept idle in the pool, so that a subsequent ++ /// request to the same origin can reuse it and skip connection setup. ++ /// ++ /// If an idle connection to the origin already exists, it is reused and no new ++ /// connection is opened. The connection is subject to the pool's idle timeout and ++ /// has no effect if pooling is disabled. ++ /// ++ /// Use [`RequestBuilder::preconnect`] to preconnect with per-request options such as ++ /// a proxy or emulation, which must match the options of the later request for the ++ /// connection to be reused. ++ /// ++ /// # Errors ++ /// ++ /// This method fails if the URI is invalid or the connection cannot be established. ++ #[inline] ++ pub fn preconnect(&self, uri: U) -> impl Future> { ++ self.get(uri).preconnect() ++ } ++ ++ pub(crate) fn execute_preconnect( ++ &self, ++ request: Request, ++ ) -> impl Future> + Send + 'static + use<> { ++ let req = http::Request::::from(request); ++ let client = self.0.clone(); ++ async move { ++ let preconnector = &client.preconnector; ++ let uri = req.uri().clone(); ++ ++ if !(uri.is_http() || uri.is_https()) || (preconnector.https_only && !uri.is_https()) { ++ return Err(Error::uri_bad_scheme(uri)); ++ } ++ ++ let (total_timeout, _) = fetch_timeout_options(&preconnector.timeout, req.extensions()); ++ let connect = preconnector.http.preconnect(req); ++ let res = match total_timeout { ++ Some(timeout) => { ++ let sleep = wreq_proto::rt::Timer::sleep(&preconnector.timer, timeout); ++ match futures_util::future::select(connect, sleep).await { ++ futures_util::future::Either::Left((res, _)) => res, ++ futures_util::future::Either::Right(_) => { ++ Err(Error::request(error::TimedOut).into()) ++ } ++ } ++ } ++ None => connect.await, ++ }; ++ ++ res.map_err(|err| { ++ let err = err ++ .downcast::() ++ .map_or_else(Error::request, |err| *err); ++ if err.uri().is_none() { err.with_uri(uri) } else { err } ++ }) + } + } + } +@@ -588,6 +668,13 @@ impl ClientBuilder { + .build(connector) + }; + ++ let preconnector = Preconnector { ++ http: service.clone(), ++ https_only: config.https_only, ++ timer: config.timer.clone(), ++ timeout: RequestConfig::new(Some(config.timeout_options)), ++ }; ++ + // Configured client service with layers + let client = { + let service = ServiceBuilder::new() +@@ -645,7 +732,10 @@ impl ClientBuilder { + } + }; + +- Ok(Client(Arc::new(client))) ++ Ok(Client(Arc::new(ClientRef { ++ service: client, ++ preconnector, ++ }))) + } + + // Runtime options +diff --git a/src/client/layer/client.rs b/src/client/layer/client.rs +index 8b000486..8e5177a2 100644 +--- a/src/client/layer/client.rs ++++ b/src/client/layer/client.rs +@@ -145,32 +145,50 @@ where + B::Data: Send, + B::Error: Into, + { +- fn request( +- &self, +- mut req: Request, +- ) -> BoxFuture<'static, Result, BoxError>> { ++ fn request(&self, mut req: Request) -> BoxFuture<'static, Result, BoxError>> { ++ match self.prepare(&mut req) { ++ Ok((this, descriptor)) => { ++ Box::pin(this.send_request(req, descriptor).map_err(Into::into)) ++ } ++ Err(err) => Box::pin(future::err(err.into())), ++ } ++ } ++ ++ /// Establishes (or reuses) a pooled connection for the request's destination ++ /// without sending the request, leaving the connection idle in the pool. ++ /// ++ /// The request is only used to derive the connection descriptor, so per-request ++ /// options (proxy, TLS/emulation, HTTP version, ...) produce the same pool key a ++ /// subsequent real request would use. ++ pub(crate) fn preconnect(&self, mut req: Request) -> BoxFuture<'static, Result<(), BoxError>> { ++ match self.prepare(&mut req) { ++ Ok((this, descriptor)) => Box::pin(async move { ++ // Dropping the pooled connection returns it to the pool as idle. ++ this.connection_for(descriptor).await.map(drop).map_err(Into::into) ++ }), ++ Err(err) => Box::pin(future::err(err.into())), ++ } ++ } ++ ++ #[allow(clippy::result_large_err)] ++ fn prepare(&self, req: &mut Request) -> Result<(Self, ConnectionDescriptor), Error> { + let is_http_connect = req.method() == Method::CONNECT; + // Validate HTTP version early + match req.version() { + Version::HTTP_10 if is_http_connect => { + warn!("CONNECT is not allowed for HTTP/1.0"); +- return Box::pin(future::err(e!(UserUnsupportedRequestMethod).into())); ++ return Err(e!(UserUnsupportedRequestMethod)); + } + Version::HTTP_10 | Version::HTTP_11 | Version::HTTP_2 => {} + // completely unsupported HTTP version (like HTTP/0.9)! + _unsupported => { + warn!("Request has unsupported version: {:?}", _unsupported); +- return Box::pin(future::err(e!(UserUnsupportedVersion).into())); ++ return Err(e!(UserUnsupportedVersion)); + } + }; + + // Extract and normalize URI +- let uri = match normalize_uri(&mut req, is_http_connect) { +- Ok(uri) => uri, +- Err(err) => { +- return Box::pin(future::err(e!(UserAbsoluteUriRequired, err).into())); +- } +- }; ++ let uri = normalize_uri(req, is_http_connect).map_err(|err| e!(UserAbsoluteUriRequired, err))?; + + let mut this = self.clone(); + +@@ -196,7 +214,7 @@ where + ConnectionDescriptor::new(uri, group, proxy, version, tls_options, socket_bind_options) + }; + +- Box::pin(this.send_request(req, descriptor).map_err(Into::into)) ++ Ok((this, descriptor)) + } + + async fn send_request( +diff --git a/src/client/layer/timeout.rs b/src/client/layer/timeout.rs +index b978d75b..0d3f340c 100644 +--- a/src/client/layer/timeout.rs ++++ b/src/client/layer/timeout.rs +@@ -105,7 +105,7 @@ where + } + } + +-fn fetch_timeout_options( ++pub(crate) fn fetch_timeout_options( + opts: &RequestConfig, + extensions: &http::Extensions, + ) -> (Option, Option) { +diff --git a/src/client/request.rs b/src/client/request.rs +index 0d4421ca..aeeb0cfd 100644 +--- a/src/client/request.rs ++++ b/src/client/request.rs +@@ -835,6 +835,21 @@ impl RequestBuilder { + } + } + ++ /// Establishes a connection for this request ahead of time, without sending it. ++ /// ++ /// Only connection-level options (such as proxy, emulation/TLS options, HTTP ++ /// version and timeout) are taken into account; the method, headers and body are ++ /// ignored. See [`Client::preconnect`] for details. ++ /// ++ /// # Errors ++ /// ++ /// This method fails if the request is invalid or the connection cannot be ++ /// established. ++ pub fn preconnect(self) -> impl Future> + Send + 'static { ++ let fut = self.request.map(|req| self.client.execute_preconnect(req)); ++ async move { fut?.await } ++ } ++ + /// Attempt to clone the RequestBuilder. + /// + /// `None` is returned if the RequestBuilder can not be cloned, From 6766268521aa9d0ec18c949c1befaedc68c867be Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Thu, 5 Nov 2026 13:51:38 +0800 Subject: [PATCH 3/4] Add Wreq::Client#preconnect Expose preconnect(url, opts = {}) to warm up a connection to an origin without sending a request, so a later request can skip DNS, TCP, proxy and TLS setup. Request options such as proxy, emulation and timeout are honored so the warmed connection matches the later request. The call releases the GVL and is interruptible via cancel and thread interrupts. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ext/wreq_rb/src/client.rs | 37 +++++++++++ test/preconnect_test.rb | 133 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 170 insertions(+) create mode 100644 test/preconnect_test.rb diff --git a/ext/wreq_rb/src/client.rs b/ext/wreq_rb/src/client.rs index 9e76dda..5fc2c6b 100644 --- a/ext/wreq_rb/src/client.rs +++ b/ext/wreq_rb/src/client.rs @@ -441,6 +441,42 @@ impl Client { self.execute_method("OPTIONS", args) } + /// client.preconnect(url) or client.preconnect(url, opts) + /// + /// Opens a connection (TCP + proxy tunnel + TLS handshake) to the URL's origin + /// and keeps it idle in the pool without sending a request. + fn preconnect(&self, args: &[Value]) -> Result<(), magnus::Error> { + let url: String = match args.first() { + Some(v) => TryConvert::try_convert(*v)?, + None => return Err(generic_error("url is required")), + }; + + let mut req = self.inner.get(&url); + if let Some(opts) = args.get(1) { + req = apply_request_options(req, &RHash::try_convert(*opts)?)?; + } + + let client_token = self.cancel_token.lock().unwrap_or_else(|e| e.into_inner()).clone(); + + let outcome: Option> = unsafe { + without_gvl(|thread_token| { + runtime().block_on(async { + tokio::select! { + biased; + _ = thread_token.cancelled() => None, + _ = client_token.cancelled() => None, + res = req.preconnect() => Some(res), + } + }) + }) + }?; + + match outcome { + Some(res) => res.map_err(to_magnus_error), + None => Err(generic_error("request interrupted")), + } + } + fn cancel(&self) { // Replace the cancel token first so new requests use a fresh token, // then cancel the old one to unblock all current in-flight select!s. @@ -870,6 +906,7 @@ pub fn init(_ruby: &magnus::Ruby, module: &magnus::RModule) -> Result<(), magnus client_class.define_method("head", method!(Client::head, -1))?; client_class.define_method("options", method!(Client::options, -1))?; client_class.define_method("request_batch", method!(Client::request_batch, -1))?; + client_class.define_method("preconnect", method!(Client::preconnect, -1))?; client_class.define_method("cancel", method!(Client::cancel, 0))?; module.define_module_function("get", function!(wreq_get, -1))?; diff --git a/test/preconnect_test.rb b/test/preconnect_test.rb new file mode 100644 index 0000000..10d7ed5 --- /dev/null +++ b/test/preconnect_test.rb @@ -0,0 +1,133 @@ +# frozen_string_literal: true + +require_relative "test_helper" +require "socket" + +class PreconnectTest < Minitest::Test + def setup + @server = TCPServer.new("127.0.0.1", 0) + @connections = Queue.new + @requests = Queue.new + @server_thread = Thread.new do + loop do + conn = @server.accept + @connections << conn + Thread.new(conn) { |c| serve_keep_alive(c) } + end + rescue IOError, Errno::EBADF + end + @url = "http://127.0.0.1:#{@server.addr[1]}/" + end + + def teardown + @server.close + @server_thread.join(2) + @connections.size.times { @connections.pop.close rescue nil } + end + + def test_preconnect_opens_connection_without_sending_a_request + client = Wreq::Client.new(emulation: false, no_proxy: true) + assert_nil client.preconnect(@url) + wait_for { @connections.size == 1 } + assert_equal 0, @requests.size, "preconnect must not send a request" + end + + def test_request_reuses_preconnected_connection + client = Wreq::Client.new(emulation: false, no_proxy: true) + client.preconnect(@url) + wait_for { @connections.size == 1 } + + resp = client.get(@url) + assert_equal 200, resp.status + assert_equal 1, @connections.size, "request should reuse the preconnected connection" + assert_equal 1, @requests.size + end + + def test_repeated_preconnect_reuses_idle_connection + client = Wreq::Client.new(emulation: false, no_proxy: true) + 3.times { client.preconnect(@url) } + client.get(@url) + assert_equal 1, @connections.size + end + + def test_preconnect_with_matching_proxy_option_is_reused + client = Wreq::Client.new(emulation: false, no_proxy: true) + proxy = @url.chomp("/") + client.preconnect("http://example.invalid/", proxy: proxy) + wait_for { @connections.size == 1 } + + resp = client.get("http://example.invalid/", proxy: proxy) + assert_equal 200, resp.status + assert_equal 1, @connections.size + assert_equal "GET http://example.invalid/ HTTP/1.1", @requests.pop + end + + def test_preconnect_connection_refused_raises + port = @server.addr[1] + @server.close + client = Wreq::Client.new(emulation: false, no_proxy: true) + assert_raises(Wreq::Error) { client.preconnect("http://127.0.0.1:#{port}/") } + end + + def test_preconnect_rejects_unsupported_scheme + client = Wreq::Client.new(emulation: false, no_proxy: true) + assert_raises(Wreq::Error) { client.preconnect("ftp://127.0.0.1/") } + end + + def test_preconnect_requires_url + client = Wreq::Client.new + assert_raises(Wreq::Error) { client.preconnect } + end + + def test_preconnect_performs_tls_handshake_and_honors_timeout + client = Wreq::Client.new(emulation: false, no_proxy: true) + started = Process.clock_gettime(Process::CLOCK_MONOTONIC) + assert_raises(Wreq::Error) do + client.preconnect("https://127.0.0.1:#{@server.addr[1]}/", timeout: 0.3) + end + elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started + assert_operator elapsed, :<, 2 + + wait_for { @connections.size == 1 } + assert_equal 22, @requests.pop.getbyte(0), "expected a TLS handshake record" + end + + def test_preconnect_https + client = Wreq::Client.new + client.preconnect("https://httpbun.com/") + resp = client.get("https://httpbun.com/get") + assert_equal 200, resp.status + end + + private + + # Answers HTTP/1.1 requests on a persistent connection, recording each request + # line. Non-HTTP input (e.g. a TLS ClientHello) is recorded and left unanswered. + def serve_keep_alive(conn) + loop do + line = conn.gets + break if line.nil? + unless line.start_with?("GET ") + @requests << line + sleep + end + @requests << line.chomp + loop do + header = conn.gets + break if header.nil? || header.chomp.empty? + end + conn.write "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: keep-alive\r\n\r\nok" + end + rescue IOError, Errno::ECONNRESET, Errno::EPIPE + ensure + conn.close rescue nil + end + + def wait_for(timeout: 2) + deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout + until yield + flunk "condition not met within #{timeout}s" if Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline + sleep 0.01 + end + end +end From 402baf943a9e2eb6aff1953b4f4e3c8a771e149d Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Thu, 5 Nov 2026 13:51:38 +0800 Subject: [PATCH 4/4] Document Client#preconnect in README Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- README.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/README.md b/README.md index be0d1ce..5fe0f54 100644 --- a/README.md +++ b/README.md @@ -142,6 +142,33 @@ client.request_batch([ - `cancel` and Ruby thread interrupts abort the whole batch, raising `Wreq::Error` with `"request interrupted"`. +### Preconnecting + +`Wreq::Client#preconnect` opens a connection to a URL's origin ahead of time — +DNS, TCP, proxy tunnel and TLS handshake — without sending a request. The +connection is kept idle in the client's pool, so the next request to the same +origin skips connection setup. + +```ruby +client = Wreq::Client.new + +client.preconnect("https://api.example.com") # returns nil, raises Wreq::Error on failure +# ... later ... +client.get("https://api.example.com/data") # reuses the warm connection +``` + +- Connection-level options must match the later request for the connection to + be reused. Pass `proxy:` and `emulation:` (plus `emulation_os:` etc.) the same + way you would to the request. `timeout:` bounds the preconnect. Other request + options are ignored. +- If an idle connection to the origin already exists, it's reused and no new + connection is opened. Preconnect several origins concurrently from separate + threads, because the GVL is released. +- The connection is subject to the pool's idle timeout (90 seconds) and may + also be closed by the server. If it's closed, the request falls back to a + fresh connection. Preconnecting has no effect with `pool_max_idle_per_host: 0`. +- `cancel` and Ruby thread interrupts abort a pending preconnect. + ### Cancelling Requests Call `cancel` on a client to interrupt all in-flight requests immediately: