diff --git a/README.md b/README.md index be0d1ce..2edaa6a 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,10 @@ client = Wreq::Client.new( tcp_nodelay: true, # disable Nagle algorithm (default: true) tcp_keepalive: 15, # SO_KEEPALIVE interval in seconds (default: 15) local_address: "1.2.3.4", # bind outgoing connections to this source IP + resolve: { # override DNS resolution per domain + "example.com" => "1.2.3.4", + "api.example.com" => ["1.2.3.4:8443", "[::1]:8443"], + }, tls_sni: true, # send SNI in TLS handshake (default: true) min_tls_version: "tls1.2", # minimum TLS version: tls1.0, tls1.1, tls1.2, tls1.3 max_tls_version: "tls1.3", # maximum TLS version @@ -100,6 +104,62 @@ resp = client.get("https://api.example.com/data") resp = client.post("https://api.example.com/data", json: { key: "value" }) ``` +## Overriding DNS + +Pass `resolve` to point specific domains at IP addresses of your choice, +bypassing DNS for them (similar to curl's `--resolve`, except that an entry +covers every port of the domain). Useful for hitting a staging host by IP while +keeping the real `Host` header and TLS SNI, or for pinning a domain to one node +of a load balancer. + +```ruby +client = Wreq::Client.new( + resolve: { + "example.com" => "127.0.0.1", # one address + "api.example.com" => ["10.0.0.1", "10.0.0.2"], # tried in order + } +) + +resp = client.get("https://example.com/") # connects to 127.0.0.1:443 +``` + +Keys are bare domain names, without a scheme, port or path. They match the +URL's host regardless of case or a trailing dot, so `https://EXAMPLE.com./` +also connects to `127.0.0.1`. + +Addresses are IPv4 or IPv6, with an optional port: + +| Value | Connects to | +|-------|-------------| +| `"1.2.3.4"` | `1.2.3.4` on the URL's port, or the scheme default | +| `"1.2.3.4:8443"` | `1.2.3.4:8443` | +| `"::1"` / `"[::1]"` | `[::1]` on the URL's port, or the scheme default | +| `"[::1]:8443"` | `[::1]:8443` | + +An IPv6 address with a port **must** be bracketed: `"::1:8080"` is itself a +valid IPv6 address, so it is parsed as one (with no port) rather than as `::1` +on port 8080. + +An explicit port in the request URL always wins over a port in the override. +Domains without an entry resolve normally, including the target of a redirect +(when `redirect` is enabled). + +### Proxies + +`resolve` only takes effect when the client looks the domain up itself. An HTTP +or HTTPS proxy, or a `socks4a://` or `socks5h://` proxy, receives the domain +name and does its own lookup, so the entry is ignored. This applies to proxies +from the `HTTP_PROXY`, `HTTPS_PROXY` and `ALL_PROXY` environment variables too, +which the client uses unless you pass `no_proxy: true`. `socks4://` and +`socks5://` proxies get the entry's first address, on the URL's port or the +scheme default. + +> **Pinning domains after a security check:** if you resolve and vet a domain +> yourself (for example to block SSRF or DNS rebinding) and then pin it with +> `resolve`, also pass `no_proxy: true` and leave redirects off (the default). +> Otherwise a proxy can do its own lookup, or a redirect can take the request +> to a domain you never checked. + ## HTTP Methods All methods are available on both `Wreq` (module-level) and `Wreq::Client` (instance-level): diff --git a/ext/wreq_rb/src/client.rs b/ext/wreq_rb/src/client.rs index 9e76dda..0ce1723 100644 --- a/ext/wreq_rb/src/client.rs +++ b/ext/wreq_rb/src/client.rs @@ -2,6 +2,7 @@ use std::ffi::c_void; use std::panic::{self, AssertUnwindSafe}; use std::ptr; use std::any::Any; +use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; @@ -13,7 +14,8 @@ use tokio::runtime::Runtime; use tokio::sync::Semaphore; use tokio::task::JoinSet; use tokio_util::sync::CancellationToken; -use std::net::IpAddr; +use std::net::{IpAddr, SocketAddr}; +use wreq::dns::{Addrs, GaiResolver, Name, Resolve, Resolving}; use wreq::header::{HeaderMap, HeaderName, HeaderValue, OrigHeaderMap}; use wreq::tls::TlsVersion; use wreq_util::{Emulation as BrowserEmulation, Platform as EmulationPlatform, Profile as BrowserProfile}; @@ -393,6 +395,14 @@ impl Client { builder = builder.local_address(addr); } + if let Some(resolve_hash) = hash_get_hash(&opts, "resolve")? { + let overrides = hash_to_dns_overrides(&resolve_hash)?; + builder = builder.dns_resolver(OverrideResolver { + overrides, + fallback: GaiResolver::new(), + }); + } + if let Some(v) = hash_get_bool(&opts, "tls_sni")? { builder = builder.tls_sni(v); } @@ -802,16 +812,20 @@ fn hash_get_array(hash: &RHash, key: &str) -> Result, magnus::Err } } +/// Convert a String or Symbol (a hash key or header name) into a String. +fn key_to_string(val: Value) -> Result { + let ruby = unsafe { Ruby::get_unchecked() }; + if val.is_kind_of(ruby.class_symbol()) { + val.funcall("to_s", ()) + } else { + TryConvert::try_convert(val) + } +} + fn array_to_orig_header_map(ary: RArray) -> Result { let mut orig = OrigHeaderMap::with_capacity(ary.len()); for elem in ary.into_iter() { - let ruby = unsafe { Ruby::get_unchecked() }; - let name_str: String = if elem.is_kind_of(ruby.class_symbol()) { - elem.funcall("to_s", ())? - } else { - TryConvert::try_convert(elem)? - }; - orig.insert(name_str); + orig.insert(key_to_string(elem)?); } Ok(orig) } @@ -822,12 +836,7 @@ fn hash_to_header_map(hash: &RHash) -> Result { if v.is_nil() { return Ok(magnus::r_hash::ForEach::Continue); } - let ruby = unsafe { Ruby::get_unchecked() }; - let ks: String = if k.is_kind_of(ruby.class_symbol()) { - k.funcall("to_s", ())? - } else { - TryConvert::try_convert(k)? - }; + let ks = key_to_string(k)?; let vs: String = v.funcall("to_s", ())?; let name = HeaderName::from_bytes(ks.as_bytes()).map_err(|e| generic_error(e))?; @@ -838,15 +847,107 @@ fn hash_to_header_map(hash: &RHash) -> Result { Ok(hmap) } +/// DNS resolver behind the `resolve:` option. wreq's own overrides +/// (`resolve_to_addrs`) compare the URL host byte for byte, so "EXAMPLE.com" +/// or "example.com." in a URL would miss an "example.com" entry and silently +/// fall back to a real DNS lookup. Here both sides are normalized first. +struct OverrideResolver { + overrides: HashMap>, + /// wreq's default resolver for the features this crate enables. + fallback: GaiResolver, +} + +impl Resolve for OverrideResolver { + fn resolve(&self, name: Name) -> Resolving { + match self.overrides.get(&normalize_host(name.as_str())) { + Some(addrs) => { + let addrs: Addrs = Box::new(addrs.clone().into_iter()); + Box::pin(std::future::ready(Ok(addrs))) + } + None => self.fallback.resolve(name), + } + } +} + +/// Host names are case-insensitive, and a trailing dot ("example.com.") names +/// the same host. +fn normalize_host(host: &str) -> String { + host.strip_suffix('.').unwrap_or(host).to_ascii_lowercase() +} + +/// Validate and normalize a `resolve:` key. Keys that can never equal a URL +/// host, such as curl-style "example.com:443" or an IP address (which is never +/// looked up), are rejected instead of being silently ignored. +fn parse_resolve_host(key: &str) -> Result { + let host = normalize_host(key); + let is_domain = !host.is_empty() + && host.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'.' | b'_')) + && host.parse::().is_err(); + if !is_domain { + return Err(generic_error(format!( + "invalid resolve host: '{}'. Use a bare domain name such as 'example.com' \ + (ASCII, without a scheme, port or path)", key + ))); + } + Ok(host) +} + +/// Parse a DNS override target into a SocketAddr. Accepts a bare IP +/// ("1.2.3.4", "::1", "[::1]") or an IP with a port ("1.2.3.4:8080", +/// "[::1]:8080"). A missing port is stored as 0, which makes wreq fall back to +/// the port in the request URL (or the scheme's default port). +/// +/// An IPv6 address with a port must be bracketed: "::1:8080" is a valid IPv6 +/// address in its own right and parses as one, not as "::1" on port 8080. +fn parse_resolve_address(host: &str, s: &str) -> Result { + if let Ok(addr) = s.parse::() { + return Ok(addr); + } + let bare = s.strip_prefix('[').and_then(|s| s.strip_suffix(']')).unwrap_or(s); + match bare.parse::() { + Ok(ip) => Ok(SocketAddr::new(ip, 0)), + Err(_) => Err(generic_error(format!( + "invalid resolve address for '{}': '{}'. Use an IP ('1.2.3.4', '::1') \ + or an IP with a port ('1.2.3.4:8080', '[::1]:8080')", host, s + ))), + } +} + +/// Convert a `resolve:` hash of host => address(es) into DNS overrides keyed by +/// normalized host. Each value is a single address string or an array of them. +fn hash_to_dns_overrides(hash: &RHash) -> Result>, magnus::Error> { + let mut overrides: HashMap> = HashMap::new(); + hash.foreach(|k: Value, v: Value| { + let key = key_to_string(k)?; + let host = parse_resolve_host(&key)?; + let addr_strs: Vec = match RArray::from_value(v) { + Some(ary) => ary.to_vec()?, + None => vec![TryConvert::try_convert(v)?], + }; + if addr_strs.is_empty() { + return Err(generic_error(format!( + "resolve entry for '{}' has no addresses", key + ))); + } + let addrs = addr_strs + .iter() + .map(|s| parse_resolve_address(&key, s)) + .collect::, _>>()?; + if overrides.insert(host, addrs).is_some() { + return Err(generic_error(format!( + "duplicate resolve host: '{}' (hosts match regardless of case \ + and a trailing dot)", key + ))); + } + Ok(magnus::r_hash::ForEach::Continue) + })?; + Ok(overrides) +} + fn hash_to_pairs(hash: &RHash) -> Result, magnus::Error> { let mut pairs: Vec<(String, String)> = Vec::new(); hash.foreach(|k: Value, v: Value| { - let ruby = unsafe { Ruby::get_unchecked() }; - let ks: String = if k.is_kind_of(ruby.class_symbol()) { - k.funcall("to_s", ())? - } else { - TryConvert::try_convert(k)? - }; + let ks = key_to_string(k)?; let vs: String = v.funcall("to_s", ())?; pairs.push((ks, vs)); Ok(magnus::r_hash::ForEach::Continue) diff --git a/test/client_test.rb b/test/client_test.rb index c19fd79..9ed88af 100644 --- a/test/client_test.rb +++ b/test/client_test.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require_relative "test_helper" +require "socket" class ClientTest < Minitest::Test def test_client_with_options @@ -57,7 +58,7 @@ def test_header_order header_order: order, headers: { "x-aaa" => "1", "x-ccc" => "2", "x-zzz" => "3" } ) - received = capture_wire_headers { |url| client.get(url) } + received = capture_request_headers { |port| client.get("http://127.0.0.1:#{port}/") }.keys positions = order.map { |h| received.index(h) }.compact assert_equal order.size, positions.size, @@ -76,7 +77,7 @@ def test_header_order_takes_precedence_over_emulation no_proxy: true, header_order: order ) - received = capture_wire_headers { |url| client.get(url) } + received = capture_request_headers { |port| client.get("http://127.0.0.1:#{port}/") }.keys positions = order.map { |h| received.index(h) }.compact assert_equal order.size, positions.size, @@ -109,10 +110,130 @@ def test_request_timeout_without_client_timeouts assert_operator elapsed, :>=, 0.4 end + def test_resolve_overrides_dns_for_a_domain + headers = capture_request_headers do |port| + client = resolving_client("wreq-resolve.invalid" => "127.0.0.1") + assert_equal 200, client.get("http://wreq-resolve.invalid:#{port}/").status + end + assert_match(/\Awreq-resolve\.invalid:/, headers["host"]) + end + + def test_resolve_honours_the_port_in_the_override + # The URL carries no explicit port, so the port from the override is used + # instead of the scheme default (80). + headers = capture_request_headers do |port| + client = resolving_client("wreq-resolve.invalid" => "127.0.0.1:#{port}") + assert_equal 200, client.get("http://wreq-resolve.invalid/").status + end + assert_equal "wreq-resolve.invalid", headers["host"] + end + + def test_resolve_accepts_multiple_addresses + dead_port = free_port + headers = capture_request_headers do |port| + # The first address refuses connections, so wreq falls through to the second. + client = resolving_client( + "wreq-resolve.invalid" => ["127.0.0.1:#{dead_port}", "127.0.0.1:#{port}"] + ) + assert_equal 200, client.get("http://wreq-resolve.invalid/", timeout: 10).status + end + assert_equal "wreq-resolve.invalid", headers["host"] + end + + def test_resolve_ignores_case_and_trailing_dot + # A differently spelled URL host must not slip past the entry to a real + # DNS lookup. The Host header keeps the URL's spelling. + [ + ["wreq-resolve.invalid", "WREQ-Resolve.invalid"], + ["wreq-resolve.invalid", "wreq-resolve.invalid."], + ["WREQ-Resolve.invalid.", "wreq-resolve.invalid"], + ].each do |key, url_host| + headers = capture_request_headers do |port| + client = resolving_client(key => "127.0.0.1:#{port}") + assert_equal 200, client.get("http://#{url_host}/").status + end + assert_equal url_host, headers["host"] + end + end + + def test_resolve_leaves_other_domains_alone + # localhost has no entry, so it resolves normally. 192.0.2.1 is reserved + # for documentation (TEST-NET-1), so applying the entry would fail. + headers = capture_request_headers do |port| + client = resolving_client("wreq-resolve.invalid" => "192.0.2.1") + assert_equal 200, client.get("http://localhost:#{port}/", timeout: 5).status + end + assert_match(/\Alocalhost:/, headers["host"]) + end + + def test_resolve_rejects_an_invalid_address + error = assert_raises(Wreq::Error) do + Wreq::Client.new(resolve: { "example.com" => "not-an-ip" }) + end + assert_match(/invalid resolve address for 'example\.com': 'not-an-ip'/, error.message) + end + + def test_resolve_rejects_hosts_that_can_never_match + ["example.com:443", "https://example.com", "1.2.3.4", "[::1]", "bücher.example", ""].each do |host| + error = assert_raises(Wreq::Error, "expected #{host.inspect} to be rejected") do + Wreq::Client.new(resolve: { host => "127.0.0.1" }) + end + assert_match(/invalid resolve host/, error.message) + end + end + + def test_resolve_rejects_hosts_that_differ_only_in_case + error = assert_raises(Wreq::Error) do + Wreq::Client.new(resolve: { "example.com" => "127.0.0.1", "EXAMPLE.com." => "127.0.0.2" }) + end + assert_match(/duplicate resolve host: 'EXAMPLE\.com\.'/, error.message) + end + private + def resolving_client(overrides) + Wreq::Client.new( + emulation: false, + http1_only: true, + no_proxy: true, + resolve: overrides + ) + end + + def free_port + server = TCPServer.new("127.0.0.1", 0) + port = server.addr[1] + server.close + port + end + + # Spins up a one-shot local HTTP/1.1 server, yields its port, and returns the + # headers of the request it received as a hash with downcased names (in wire + # order, as Ruby hashes keep insertion order). + def capture_request_headers + server = TCPServer.new("127.0.0.1", 0) + headers = {} + server_thread = Thread.new do + conn = server.accept + conn.gets # skip request line + while (line = conn.gets&.chomp) && !line.empty? + name, value = line.split(":", 2) + headers[name.downcase] = value.strip + end + conn.write "HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ensure + conn&.close + end + yield server.addr[1] + # join re-raises anything that went wrong in the server thread. + assert server_thread.join(5), "Local HTTP server did not finish handling a request" + headers + ensure + server_thread&.kill + server&.close + end + def stalled_tls_request_duration(options, request_timeout: 3) - require "socket" server = TCPServer.new("127.0.0.1", 0) first_byte = nil server_thread = Thread.new do @@ -139,30 +260,4 @@ def stalled_tls_request_duration(options, request_timeout: 3) server_thread&.join server&.close end - - # Spins up a local TCP server, yields the port formatted into a URL, captures - # the header names from the raw HTTP/1.1 request, then tears down the server. - def capture_wire_headers - require "socket" - server = TCPServer.new("127.0.0.1", 0) - port = server.addr[1] - received = [] - t = Thread.new do - conn = server.accept - conn.gets # skip request line - loop do - line = conn.gets&.chomp - break if line.nil? || line.empty? - received << line.split(":", 2).first.downcase - end - conn.write "HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" - conn.close - rescue - conn&.close - end - yield format("http://127.0.0.1:#{port}/") - t.join(5) - server.close - received - end end