diff --git a/README.md b/README.md index be0d1ce..4e87dc3 100644 --- a/README.md +++ b/README.md @@ -182,7 +182,7 @@ wreq-rb emulates real browser TLS fingerprints, HTTP/2 settings, and headers by resp = Wreq.get("https://tls.peet.ws/api/all") # Explicit browser emulation -client = Wreq::Client.new(emulation: "firefox_151") +client = Wreq::Client.new(emulation: "firefox_158") client = Wreq::Client.new(emulation: "safari_18.5") client = Wreq::Client.new(emulation: "edge_148") @@ -205,12 +205,18 @@ resp = client.get("https://example.com", emulation: "safari_26.2") | Browser | Example values | |---------|---------------| | Chrome | `chrome_100` .. `chrome_148` | -| Firefox | `firefox_109` .. `firefox_151`, `firefox_private_135` | +| Firefox | `firefox_109` .. `firefox_158`, `firefox_private_135` | | Safari | `safari_15.3` .. `safari_26.4`, `safari_ios_26`, `safari_ipad_18` | | Edge | `edge_101` .. `edge_148` | | Opera | `opera_116` .. `opera_131` | | OkHttp | `okhttp_3.9` .. `okhttp_5` | +A client created with `firefox_152` or newer also reproduces Firefox's HTTP/1.1 request: `Host` first, header names spelled and ordered the way Firefox sends them, and `Connection: keep-alive`. Headers you add yourself (`Cookie`, `Referer`, `Content-Type`, ...) go where Firefox puts them on a navigation, over HTTP/2 as well; headers Firefox has no fixed place for follow `Accept-Encoding` in the order and spelling you gave them. + +- `TE: trailers`, which Firefox sends over HTTP/2 only, is also sent over HTTP/1.1 unless the client is created with `http1_only: true`. That option makes the TLS handshake offer only `http/1.1`, which Firefox never does. +- A client's `header_order` takes precedence over the profile's order. As with every profile, a per-request `emulation:` brings its own headers on top of the client's and ignores the client's `header_order`, so create the client with `emulation: false` when you switch profiles per request. `Wreq.get(url, emulation: ...)` runs on the default Chrome client and sends its `sec-ch-ua` headers too. +- Mixing is lossy the other way round as well: a request that selects another browser's profile on a `firefox_152`+ client inherits `Connection: keep-alive`, and over HTTP/2 the order of the headers after it is not kept. + ## Response The `Wreq::Response` object provides: diff --git a/ext/wreq_rb/src/client.rs b/ext/wreq_rb/src/client.rs index 9e76dda..92013fe 100644 --- a/ext/wreq_rb/src/client.rs +++ b/ext/wreq_rb/src/client.rs @@ -16,9 +16,11 @@ use tokio_util::sync::CancellationToken; use std::net::IpAddr; use wreq::header::{HeaderMap, HeaderName, HeaderValue, OrigHeaderMap}; use wreq::tls::TlsVersion; +use wreq::IntoEmulation; use wreq_util::{Emulation as BrowserEmulation, Platform as EmulationPlatform, Profile as BrowserProfile}; use crate::error::{generic_error, to_magnus_error, wreq_error}; +use crate::firefox::FirefoxProfile; use crate::response::Response; // -------------------------------------------------------------------------- @@ -204,13 +206,26 @@ async fn execute_batch(reqs: Vec, concurrency: usize) -> V // -------------------------------------------------------------------------- /// The default browser profile to apply when none is specified. -const DEFAULT_EMULATION: BrowserProfile = BrowserProfile::Chrome149; +const DEFAULT_EMULATION: EmulationProfile = EmulationProfile::Bundled(BrowserProfile::Chrome149); + +/// A browser profile selectable through the `emulation` option. +#[derive(Clone, Copy)] +enum EmulationProfile { + /// A profile that ships with wreq-util. + Bundled(BrowserProfile), + /// A Firefox release newer than wreq-util knows about. + Firefox(&'static FirefoxProfile), +} -/// Parse a Ruby string like "chrome_143" into a BrowserProfile variant. -fn parse_emulation(name: &str) -> Result { +/// Parse a Ruby string like "chrome_143" into an emulation profile. +fn parse_emulation(name: &str) -> Result { let json_val = serde_json::Value::String(name.to_string()); - serde_json::from_value::(json_val) - .map_err(|_| generic_error(format!("unknown emulation: '{}'. Use names like 'chrome_148', 'firefox_151', 'safari_18.5', etc.", name))) + if let Ok(profile) = serde_json::from_value::(json_val) { + return Ok(EmulationProfile::Bundled(profile)); + } + FirefoxProfile::find(name) + .map(EmulationProfile::Firefox) + .ok_or_else(|| generic_error(format!("unknown emulation: '{}'. Use names like 'chrome_148', 'firefox_158', 'safari_18.5', etc.", name))) } /// Parse a Ruby string like "windows" into an EmulationPlatform variant. @@ -220,19 +235,160 @@ fn parse_emulation_os(name: &str) -> Result { .map_err(|_| generic_error("unknown emulation_os. Use: 'windows', 'macos', 'linux', 'android', 'ios'")) } -/// Build a BrowserEmulation from a profile and an optional platform from the opts hash. +/// Build the emulation for a profile on the given platform. `http1_only` says +/// that the client never negotiates HTTP/2. +fn build_emulation( + profile: EmulationProfile, + platform: EmulationPlatform, + http1_only: bool, +) -> wreq::Emulation { + match profile { + EmulationProfile::Bundled(profile) => BrowserEmulation::builder() + .profile(profile) + .platform(platform) + .build() + .into_emulation(), + EmulationProfile::Firefox(profile) => profile.emulation(platform, http1_only), + } +} + +/// Build the emulation for a profile and an optional platform from the opts hash. fn build_emulation_option( - profile: BrowserProfile, + profile: EmulationProfile, opts: &RHash, -) -> Result { + http1_only: bool, +) -> Result { let platform = match hash_get_string(opts, "emulation_os")? { Some(platform_name) => parse_emulation_os(&platform_name)?, None => EmulationPlatform::default(), }; - Ok(BrowserEmulation::builder() - .profile(profile) - .platform(platform) - .build()) + Ok(build_emulation(profile, platform, http1_only)) +} + +/// The header order of a client or request whose profile spells and orders the +/// request headers itself, which today means the local Firefox profiles. +/// +/// wreq writes the headers such an order names first and whatever is left +/// after them, in an order that follows from how its header map removes +/// entries rather than from the order the headers were given in. So the order +/// handed to wreq names every header the gem knows the request carries. +#[derive(Clone)] +struct HeaderOrder { + /// Names fixed by the `header_order` option. They come first, spelled as + /// the user spelled them. + user: OrigHeaderMap, + /// The profile's names, without the ones in `user`. + profile: OrigHeaderMap, + /// The profile name that headers neither list names are written before: + /// the profile's own slot for them, or the next name after it that + /// `user` left in `profile`. `None` puts them last. + slot: Option, + /// The names in the client's `headers` option, spelled as given. Those + /// that neither list names get a place. + extra: Vec, +} + +impl HeaderOrder { + fn new(user: Option<&OrigHeaderMap>, profile: &OrigHeaderMap, extra: Vec) -> Self { + let user = user.cloned().unwrap_or_default(); + let mut rest = OrigHeaderMap::with_capacity(profile.len()); + let mut slot = None; + let mut slot_reached = false; + for (name, spelled) in profile.iter() { + slot_reached |= *name == crate::firefox::UNLISTED_HEADERS_BEFORE; + if !user.iter().any(|(ordered, _)| ordered == name) { + rest.insert(spelled); + if slot_reached && slot.is_none() { + slot = Some(name.clone()); + } + } + } + HeaderOrder { user, profile: rest, slot, extra } + } + + fn lists(&self, name: &HeaderName) -> bool { + self.user.iter().chain(self.profile.iter()).any(|(listed, _)| listed == name) + } + + /// The headers among the client's and the request's own that the order + /// has no place for, in the order they were given. + fn unlisted<'a>(&'a self, request: &'a [String]) -> Vec<(HeaderName, &'a str)> { + let mut unlisted: Vec<(HeaderName, &str)> = Vec::new(); + for spelled in self.extra.iter().chain(request) { + let Ok(name) = HeaderName::from_bytes(spelled.as_bytes()) else { + continue; + }; + if !self.lists(&name) && !unlisted.iter().any(|(seen, _)| *seen == name) { + unlisted.push((name, spelled)); + } + } + unlisted + } + + /// Whether a request with these headers needs an order of its own, that + /// is whether it carries a header the client's order does not name. + fn misses(&self, request: &[String]) -> bool { + self.unlisted(request).len() > self.unlisted(&[]).len() + } + + /// The order for a request that carries the `request` headers on top of + /// the client's. + fn with(&self, request: &[String]) -> OrigHeaderMap { + let unlisted = self.unlisted(request); + let mut order = OrigHeaderMap::with_capacity( + self.user.len() + self.profile.len() + unlisted.len(), + ); + for (_, spelled) in self.user.iter() { + order.insert(spelled); + } + + let mut placed = false; + for (name, spelled) in self.profile.iter() { + if !placed && self.slot.as_ref() == Some(name) { + for (_, spelled) in &unlisted { + order.insert(spelled.to_string()); + } + placed = true; + } + order.insert(spelled); + } + if !placed { + for (_, spelled) in &unlisted { + order.insert(spelled.to_string()); + } + } + order + } +} + +/// Whether a client with these options only ever speaks HTTP/1.1. Both options +/// together leave an HTTP/2 client: `http2_only` is applied last and wins. +fn restricted_to_http1(http1_only: bool, http2_only: bool) -> bool { + http1_only && !http2_only +} + +/// What applying a request's options needs to know about the client, and what +/// it learns about the request's headers on the way. +struct RequestHeaders { + http1_only: bool, + /// The header order in force: the client's, or the one of a profile + /// selected for this request. + order: Option>, + /// Headers the client adds to every request, spelled as given. + client: Arc>, + /// Headers set for this request, spelled as given. + request: Vec, +} + +impl RequestHeaders { + /// Give the request an order of its own when it carries headers the + /// order in force does not name. + fn finish(self, req: wreq::RequestBuilder) -> wreq::RequestBuilder { + match &self.order { + Some(order) if order.misses(&self.request) => req.orig_headers(order.with(&self.request)), + _ => req, + } + } } // -------------------------------------------------------------------------- @@ -242,6 +398,12 @@ fn build_emulation_option( #[magnus::wrap(class = "Wreq::Client", free_immediately)] struct Client { inner: wreq::Client, + /// The client never negotiates HTTP/2 (`http1_only: true`). + http1_only: bool, + /// Set when the client's profile orders the request headers itself. + header_order: Option>, + /// The names in the client's `headers` option, spelled as given. + header_names: Arc>, cancel_token: std::sync::Mutex, } @@ -257,37 +419,68 @@ impl Client { let mut builder = wreq::Client::builder() .retry(wreq::retry::Policy::never()); + let mut http1_only = false; + let mut header_order = None; + let mut header_names = Vec::new(); + if let Some(opts) = opts { + let force_http1 = hash_get_bool(&opts, "http1_only")? == Some(true); + let force_http2 = hash_get_bool(&opts, "http2_only")? == Some(true); + http1_only = restricted_to_http1(force_http1, force_http2); + + let client_headers = match hash_get_hash(&opts, "headers")? { + Some(hdr_hash) => Some(hash_to_named_header_map(&hdr_hash)?), + None => None, + }; + if let Some((_, names)) = &client_headers { + header_names = names.clone(); + } + // Apply header_order BEFORE emulation so the user's ordering takes precedence - if let Some(ary) = hash_get_array(&opts, "header_order")? { - let orig = array_to_orig_header_map(ary)?; - builder = builder.orig_headers(orig); + let user_order = match hash_get_array(&opts, "header_order")? { + Some(ary) => Some(array_to_orig_header_map(ary)?), + None => None, + }; + if let Some(orig) = &user_order { + builder = builder.orig_headers(orig.clone()); } - if let Some(val) = hash_get_value(&opts, "emulation")? { - let ruby = unsafe { Ruby::get_unchecked() }; - if val.is_kind_of(ruby.class_false_class()) { - // emulation: false — skip emulation entirely - } else if val.is_kind_of(ruby.class_true_class()) { - let opt = build_emulation_option(DEFAULT_EMULATION, &opts)?; - builder = builder.emulation(opt); - } else { - let name: String = TryConvert::try_convert(val)?; - let emu = parse_emulation(&name)?; - let opt = build_emulation_option(emu, &opts)?; - builder = builder.emulation(opt); + let profile = match hash_get_value(&opts, "emulation")? { + Some(val) => { + let ruby = unsafe { Ruby::get_unchecked() }; + if val.is_kind_of(ruby.class_false_class()) { + // emulation: false — skip emulation entirely + None + } else if val.is_kind_of(ruby.class_true_class()) { + Some(DEFAULT_EMULATION) + } else { + let name: String = TryConvert::try_convert(val)?; + Some(parse_emulation(&name)?) + } } - } else { - let opt = build_emulation_option(DEFAULT_EMULATION, &opts)?; - builder = builder.emulation(opt); + None => Some(DEFAULT_EMULATION), + }; + if let Some(profile) = profile { + let mut emulation = build_emulation_option(profile, &opts, http1_only)?; + if let EmulationProfile::Firefox(_) = profile { + // The user's position and spelling of a header win over + // the profile's, and the client's own headers get a place. + let order = HeaderOrder::new( + user_order.as_ref(), + &emulation.orig_headers, + header_names.clone(), + ); + emulation.orig_headers = order.with(&[]); + header_order = Some(Arc::new(order)); + } + builder = builder.emulation(emulation); } if let Some(ua) = hash_get_string(&opts, "user_agent")? { builder = builder.user_agent(ua); } - if let Some(hdr_hash) = hash_get_hash(&opts, "headers")? { - let hmap = hash_to_header_map(&hdr_hash)?; + if let Some((hmap, _)) = client_headers { builder = builder.default_headers(hmap); } @@ -347,10 +540,10 @@ impl Client { builder = builder.tls_cert_verification(v); } - if let Some(true) = hash_get_bool(&opts, "http1_only")? { + if force_http1 { builder = builder.http1_only(); } - if let Some(true) = hash_get_bool(&opts, "http2_only")? { + if force_http2 { builder = builder.http2_only(); } @@ -405,11 +598,17 @@ impl Client { builder = builder.tls_max_version(parse_tls_version(&s)?); } } else { - builder = builder.emulation(DEFAULT_EMULATION); + builder = builder.emulation(build_emulation(DEFAULT_EMULATION, EmulationPlatform::default(), false)); } let client = builder.build().map_err(to_magnus_error)?; - Ok(Client { inner: client, cancel_token: std::sync::Mutex::new(CancellationToken::new()) }) + Ok(Client { + inner: client, + http1_only, + header_order, + header_names: Arc::new(header_names), + cancel_token: std::sync::Mutex::new(CancellationToken::new()), + }) } /// client.get(url) or client.get(url, opts) @@ -472,9 +671,11 @@ impl Client { let mut req = self.inner.request(method, &url); + let mut headers = self.request_headers(); if let Some(opts) = opts { - req = apply_request_options(req, &opts)?; + req = apply_request_options(req, &opts, &mut headers)?; } + req = headers.finish(req); let client_token = self.cancel_token.lock().unwrap_or_else(|e| e.into_inner()).clone(); @@ -600,13 +801,23 @@ impl Client { } let mut req = self.inner.request(method, &url); + let mut headers = self.request_headers(); if let Some(shared) = shared { - req = apply_request_options(req, shared)?; + req = apply_request_options(req, shared, &mut headers)?; } if let Some(item_opts) = item_opts { - req = apply_request_options(req, &item_opts)?; + req = apply_request_options(req, &item_opts, &mut headers)?; + } + Ok(headers.finish(req)) + } + + fn request_headers(&self) -> RequestHeaders { + RequestHeaders { + http1_only: self.http1_only, + order: self.header_order.clone(), + client: self.header_names.clone(), + request: Vec::new(), } - Ok(req) } } @@ -621,24 +832,38 @@ fn value_to_method(val: Value) -> Result { fn apply_request_options( mut req: wreq::RequestBuilder, opts: &RHash, + headers: &mut RequestHeaders, ) -> Result { if let Some(val) = hash_get_value(opts, "emulation")? { let ruby = unsafe { Ruby::get_unchecked() }; if val.is_kind_of(ruby.class_false_class()) { // emulation: false — no per-request emulation override - } else if val.is_kind_of(ruby.class_true_class()) { - let opt = build_emulation_option(DEFAULT_EMULATION, opts)?; - req = req.emulation(opt); } else { - let name: String = TryConvert::try_convert(val)?; - let emu = parse_emulation(&name)?; - let opt = build_emulation_option(emu, opts)?; + let profile = if val.is_kind_of(ruby.class_true_class()) { + DEFAULT_EMULATION + } else { + let name: String = TryConvert::try_convert(val)?; + parse_emulation(&name)? + }; + let mut opt = build_emulation_option(profile, opts, headers.http1_only)?; + // A profile selected for the request replaces the client's header + // order with its own, which is empty for the bundled profiles. + headers.order = match profile { + EmulationProfile::Firefox(_) => { + // The client's own headers still go out with the request. + let order = HeaderOrder::new(None, &opt.orig_headers, headers.client.to_vec()); + opt.orig_headers = order.with(&[]); + Some(Arc::new(order)) + } + EmulationProfile::Bundled(_) => None, + }; req = req.emulation(opt); } } if let Some(hdr_hash) = hash_get_hash(opts, "headers")? { - let hmap = hash_to_header_map(&hdr_hash)?; + let (hmap, names) = hash_to_named_header_map(&hdr_hash)?; + headers.request.extend(names); req = req.headers(hmap); } @@ -816,8 +1041,11 @@ fn array_to_orig_header_map(ary: RArray) -> Result Ok(orig) } -fn hash_to_header_map(hash: &RHash) -> Result { +/// Convert a headers hash into a header map, along with the names as the +/// caller spelled them, in the order given. +fn hash_to_named_header_map(hash: &RHash) -> Result<(HeaderMap, Vec), magnus::Error> { let mut hmap = HeaderMap::new(); + let mut names = Vec::new(); hash.foreach(|k: Value, v: Value| { if v.is_nil() { return Ok(magnus::r_hash::ForEach::Continue); @@ -833,9 +1061,10 @@ fn hash_to_header_map(hash: &RHash) -> Result { HeaderName::from_bytes(ks.as_bytes()).map_err(|e| generic_error(e))?; let value = HeaderValue::from_str(&vs).map_err(|e| generic_error(e))?; hmap.insert(name, value); + names.push(ks); Ok(magnus::r_hash::ForEach::Continue) })?; - Ok(hmap) + Ok((hmap, names)) } fn hash_to_pairs(hash: &RHash) -> Result, magnus::Error> { @@ -885,6 +1114,96 @@ pub fn init(_ruby: &magnus::Ruby, module: &magnus::RModule) -> Result<(), magnus #[cfg(test)] mod tests { use super::*; + + fn firefox_order() -> OrigHeaderMap { + FirefoxProfile::find("firefox_158") + .unwrap() + .emulation(EmulationPlatform::default(), false) + .orig_headers + } + + fn order_of(names: &[&'static str]) -> OrigHeaderMap { + let mut order = OrigHeaderMap::new(); + for name in names { + order.insert(*name); + } + order + } + + fn spelled(order: &OrigHeaderMap) -> Vec { + order + .iter() + .map(|(_, name)| String::from_utf8(name.as_ref().to_vec()).unwrap()) + .collect() + } + + fn strings(names: &[&str]) -> Vec { + names.iter().map(|name| name.to_string()).collect() + } + + fn position(order: &[String], name: &str) -> usize { + order + .iter() + .position(|listed| listed == name) + .unwrap_or_else(|| panic!("{name} missing from {order:?}")) + } + + #[test] + fn header_order_without_extras_is_the_profile_order() { + let order = HeaderOrder::new(None, &firefox_order(), Vec::new()); + assert_eq!(spelled(&order.with(&[])), spelled(&firefox_order())); + assert!(!order.misses(&[])); + } + + #[test] + fn header_order_places_unknown_headers_before_content_type_in_the_order_given() { + let order = HeaderOrder::new(None, &firefox_order(), strings(&["X-Client", "cookie"])); + let names = spelled(&order.with(&strings(&["x-b", "X-A", "Referer", "x-client"]))); + + let start = position(&names, "Range") + 1; + assert_eq!(names[start..start + 4], ["X-Client", "x-b", "X-A", "Content-Type"]); + assert_eq!(names.len(), spelled(&firefox_order()).len() + 3); + assert_eq!(names.last().unwrap(), "TE"); + } + + #[test] + fn header_order_needs_a_request_order_only_for_new_names() { + let order = HeaderOrder::new(None, &firefox_order(), strings(&["X-Client"])); + assert!(!order.misses(&strings(&["cookie", "REFERER", "x-client"]))); + assert!(order.misses(&strings(&["cookie", "x-new"]))); + assert!(!order.misses(&strings(&["not a header name"]))); + } + + #[test] + fn header_order_puts_the_users_names_first_with_their_spelling() { + let user = order_of(&["host", "ACCEPT"]); + let order = HeaderOrder::new(Some(&user), &firefox_order(), Vec::new()); + let names = spelled(&order.with(&strings(&["X-A"]))); + + assert_eq!(names[..4], ["host", "ACCEPT", "User-Agent", "Accept-Language"]); + assert_eq!(names.iter().filter(|name| name.eq_ignore_ascii_case("accept")).count(), 1); + assert!(position(&names, "X-A") + 1 == position(&names, "Content-Type")); + } + + #[test] + fn header_order_keeps_unknown_headers_ahead_of_the_rest_when_the_user_orders_content_type() { + let user = order_of(&["Content-Type", "content-length"]); + let order = HeaderOrder::new(Some(&user), &firefox_order(), Vec::new()); + let names = spelled(&order.with(&strings(&["X-A", "X-B"]))); + + assert_eq!(names[..2], ["Content-Type", "content-length"]); + let start = position(&names, "X-A"); + assert_eq!(names[start - 1..start + 3], ["Range", "X-A", "X-B", "Origin"]); + assert_eq!(names.last().unwrap(), "TE"); + } + + #[test] + fn http2_only_wins_over_http1_only() { + assert!(restricted_to_http1(true, false)); + assert!(!restricted_to_http1(true, true)); + assert!(!restricted_to_http1(false, true)); + assert!(!restricted_to_http1(false, false)); + } use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; static READY: AtomicBool = AtomicBool::new(false); diff --git a/ext/wreq_rb/src/firefox.rs b/ext/wreq_rb/src/firefox.rs new file mode 100644 index 0000000..3ea1055 --- /dev/null +++ b/ext/wreq_rb/src/firefox.rs @@ -0,0 +1,575 @@ +//! Firefox profiles newer than the ones shipped with the pinned `wreq-util`. +//! +//! `wreq-util` 0.2.0 stops at Firefox 151, so the releases after it are defined +//! here: each one starts from the bundled Firefox 151 profile and overrides the +//! parts that differ on the wire. A name that `wreq-util` knows always resolves +//! to its profile, so once it ships one of these releases the entry here is +//! dead and should be removed (a unit test fails when that happens). +//! +//! The bundled Firefox 151 profile is itself slightly off, so three corrections +//! to the TLS and HTTP/2 fingerprint apply to every profile in this module +//! (upstream `wreq-util` has since made the same ones on its main branch): +//! +//! - `TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA` is no longer offered; +//! - `Accept-Language` is `en-US,en;q=0.9` rather than `q=0.5` (Bugzilla 2000765); +//! - the HEADERS frame carries priority weight 42 rather than 22. Firefox +//! derives the weight from the request's urgency: 42 goes with the +//! navigation request these headers describe (`priority: u=0, i`), 22 with +//! lower-urgency requests such as the favicon. +//! +//! The bundled profile also leaves the HTTP/1.1 request to the HTTP stack, +//! which writes lower-case header names with `host` last. The profiles here +//! carry the names as Firefox spells and orders them ([`HEADER_ORDER`]) and +//! add `Connection: keep-alive`, which only reaches the wire over HTTP/1.1. +//! +//! On top of that the cipher suites and groups changed twice between 152 and +//! 158. The versions come from the Firefox sources at the +//! `FIREFOX__0_RELEASE` tags: +//! +//! - 154 turned off `security.ssl3.ecdhe_ecdsa_aes_256_sha` on release builds +//! (`StaticPrefList.yaml`), dropping `TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA`; +//! - 156 stopped advertising `ffdhe2048`/`ffdhe3072` in `supported_groups` +//! unless a DHE cipher suite is enabled, and none is (`nsNSSIOLayer.cpp`). +//! +//! Every profile was compared against the official macOS release build of the +//! same version (152.0 to 158.0) making a top-level navigation. JA3, JA4, the +//! Akamai HTTP/2 fingerprint, the cipher suites, the extension order and +//! bodies, groups, key share groups, signature algorithms, the HTTP/2 +//! SETTINGS, WINDOW_UPDATE and HEADERS priority, the header names, values and +//! order, and the HTTP/1.1 request all match. 157 hard-coded the P-256 key +//! share and added ML-DSA signature schemes behind `security.tls.enable_mldsa`, +//! which is off, so its ClientHello is the same as 156's. +//! +//! Known gaps. On a desktop platform, and with HTTP/2 left on, none of them is +//! visible in JA3, JA4 or the Akamai fingerprint: +//! +//! - The GREASE ECH extension. Firefox sends a fixed payload of 239 bytes up +//! to 154 and 240 from 155 (NSS 3.127), while BoringSSL picks 144, 176, 208 +//! or 240 at random, so the ClientHello length differs. Firefox also fills +//! `enc` with random bytes where BoringSSL sends a real X25519 public key. +//! - Firefox reuses its X25519 key inside the X25519MLKEM768 key share; +//! BoringSSL generates two independent keys. +//! - HTTP/2: Firefox sends a WINDOW_UPDATE for the stream right after HEADERS +//! and never encodes `:path` as an indexed header field; the weight is the +//! same for every request (see above). +//! - HTTP/1.1: `TE: trailers` is sent as well, also to `http://` URLs, where +//! Firefox sends it over HTTP/2 only. A client created with `http1_only` +//! leaves it out, but such a client offers only `http/1.1` in ALPN, which +//! changes JA4. Over TLS it is one or the other. +//! - The header order is that of a navigation. Firefox orders a `fetch()` +//! request differently: `Referer` moves up to follow `Accept-Encoding`. The +//! `header_order` client option takes precedence over the profile's order. +//! A profile chosen for a single request brings its own order and, like +//! every per-request profile, ignores the client's `header_order`. +//! - The navigation headers are always sent. A link click or a form +//! submission made by a script has no `Sec-Fetch-User`, for instance, and +//! there is no way to leave a default header out. +//! - These are desktop fingerprints. Android and iOS platforms only swap the +//! User-Agent, although Firefox for Android uses different HTTP/2 settings. + +use std::borrow::Cow; + +use wreq::header::{ + HeaderName, HeaderValue, OrigHeaderMap, ACCEPT_LANGUAGE, CONNECTION, CONTENT_TYPE, TE, USER_AGENT, +}; +use wreq::http2::{StreamDependency, StreamId}; +use wreq::{Group, IntoEmulation}; +use wreq_util::{Emulation as BrowserEmulation, Platform, Profile as BrowserProfile}; + +/// The bundled profile every profile in this module is derived from. +const BASE_PROFILE: BrowserProfile = BrowserProfile::Firefox151; + +/// The version as it appears in [`BASE_PROFILE`]'s User-Agent strings. +const BASE_VERSION: &str = "151.0"; + +const ACCEPT_LANGUAGE_VALUE: &str = "en-US,en;q=0.9"; + +/// Weight of the HEADERS frame priority, one less than the value on the wire. +const HEADERS_PRIORITY_WEIGHT: u8 = 41; + +/// Request header names in the order Firefox sends them, spelled the way it +/// writes them in an HTTP/1.1 request. +/// +/// The order is that of a top-level navigation, recorded from Firefox 152 and +/// 158 over HTTP/1.1: a plain GET, a GET carrying a referrer, cookies and +/// cache validators, a reload and a form POST. HTTP/2 uses the same order +/// without `Host` and `Connection` and ends with `TE` (recorded for the GETs +/// and the form POST). Two groups of names do not come from those recordings: +/// +/// - `Authorization` and `Range` were only seen on `fetch()` requests, which +/// put `Referer` and then the headers the script sets, in the script's +/// order, right after `Accept-Encoding`. Their place here is the same one +/// for a request without a referrer. The `Authorization` header Firefox +/// adds itself after an HTTP authentication challenge comes later, right +/// before `Connection` (`nsHttpChannel.cpp`). +/// - `DNT` and `Sec-GPC` follow the Firefox sources, which add them between +/// `Origin` and `Connection` (`nsHttpChannel.cpp`). +/// +/// A header that is not listed is written right before +/// [`UNLISTED_HEADERS_BEFORE`], spelled as the caller spelled it. If the +/// `header_order` option claims that name, it is written before the next +/// listed name the option leaves alone. +const HEADER_ORDER: &[&str] = &[ + "Host", + "User-Agent", + "Accept", + "Accept-Language", + "Accept-Encoding", + "Authorization", + "Range", + "Content-Type", + "Content-Length", + "Origin", + "DNT", + "Sec-GPC", + "Connection", + "Referer", + "Cookie", + "Upgrade-Insecure-Requests", + "Sec-Fetch-Dest", + "Sec-Fetch-Mode", + "Sec-Fetch-Site", + "Sec-Fetch-User", + "If-Modified-Since", + "If-None-Match", + "Priority", + "Pragma", + "Cache-Control", + "TE", +]; + +/// Headers that [`HEADER_ORDER`] does not name go right before this one, in +/// the order the caller gave them: the place where Firefox writes the headers +/// a script passes to `fetch()`. +pub const UNLISTED_HEADERS_BEFORE: HeaderName = CONTENT_TYPE; + +/// Cipher suites offered by Firefox 152 and 153. +const CIPHERS_152: &str = concat!( + "TLS_AES_128_GCM_SHA256:", + "TLS_CHACHA20_POLY1305_SHA256:", + "TLS_AES_256_GCM_SHA384:", + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256:", + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256:", + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256:", + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256:", + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384:", + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384:", + "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA:", + "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA:", + "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA:", + "TLS_RSA_WITH_AES_128_GCM_SHA256:", + "TLS_RSA_WITH_AES_256_GCM_SHA384:", + "TLS_RSA_WITH_AES_128_CBC_SHA:", + "TLS_RSA_WITH_AES_256_CBC_SHA", +); + +/// Cipher suites offered since Firefox 154, which dropped +/// `TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA`. +const CIPHERS_154: &str = concat!( + "TLS_AES_128_GCM_SHA256:", + "TLS_CHACHA20_POLY1305_SHA256:", + "TLS_AES_256_GCM_SHA384:", + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256:", + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256:", + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256:", + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256:", + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384:", + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384:", + "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA:", + "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA:", + "TLS_RSA_WITH_AES_128_GCM_SHA256:", + "TLS_RSA_WITH_AES_256_GCM_SHA384:", + "TLS_RSA_WITH_AES_128_CBC_SHA:", + "TLS_RSA_WITH_AES_256_CBC_SHA", +); + +/// Supported groups advertised up to Firefox 155. +const GROUPS_152: &str = "X25519MLKEM768:X25519:P-256:P-384:P-521:ffdhe2048:ffdhe3072"; + +/// Supported groups advertised since Firefox 156, which dropped the FFDHE groups. +const GROUPS_156: &str = "X25519MLKEM768:X25519:P-256:P-384:P-521"; + +/// A Firefox release that `wreq-util` has no profile for. +#[derive(Debug)] +pub struct FirefoxProfile { + /// The name accepted by the `emulation` option, e.g. `firefox_158`. + name: &'static str, + /// The version as it appears in the User-Agent, e.g. `158.0`. + version: &'static str, + cipher_list: &'static str, + curves_list: &'static str, +} + +const PROFILES: &[FirefoxProfile] = &[ + FirefoxProfile { + name: "firefox_152", + version: "152.0", + cipher_list: CIPHERS_152, + curves_list: GROUPS_152, + }, + FirefoxProfile { + name: "firefox_153", + version: "153.0", + cipher_list: CIPHERS_152, + curves_list: GROUPS_152, + }, + FirefoxProfile { + name: "firefox_154", + version: "154.0", + cipher_list: CIPHERS_154, + curves_list: GROUPS_152, + }, + FirefoxProfile { + name: "firefox_155", + version: "155.0", + cipher_list: CIPHERS_154, + curves_list: GROUPS_152, + }, + FirefoxProfile { + name: "firefox_156", + version: "156.0", + cipher_list: CIPHERS_154, + curves_list: GROUPS_156, + }, + FirefoxProfile { + name: "firefox_157", + version: "157.0", + cipher_list: CIPHERS_154, + curves_list: GROUPS_156, + }, + FirefoxProfile { + name: "firefox_158", + version: "158.0", + cipher_list: CIPHERS_154, + curves_list: GROUPS_156, + }, +]; + +impl FirefoxProfile { + /// Look up a profile by its `emulation` name. + pub fn find(name: &str) -> Option<&'static FirefoxProfile> { + PROFILES.iter().find(|profile| profile.name == name) + } + + /// Build the emulation for this release on the given platform. + /// + /// `http1_only` says that the client is restricted to HTTP/1.1. Firefox + /// sends `TE: trailers` over HTTP/2 only, but the default headers cannot + /// depend on the protocol a connection ends up with, so the header is + /// left out only for such a client. + pub fn emulation(&self, platform: Platform, http1_only: bool) -> wreq::Emulation { + let base = BrowserEmulation::builder() + .profile(BASE_PROFILE) + .platform(platform) + .build() + .into_emulation(); + + let mut headers = base.headers; + let user_agent = headers + .get(USER_AGENT) + .and_then(|value| value.to_str().ok()) + .map(|value| value.replace(BASE_VERSION, self.version)) + .and_then(|value| HeaderValue::from_str(&value).ok()); + if let Some(user_agent) = user_agent { + headers.insert(USER_AGENT, user_agent); + } + headers.insert( + ACCEPT_LANGUAGE, + HeaderValue::from_static(ACCEPT_LANGUAGE_VALUE), + ); + + // HTTP/2 forbids the header and the HTTP/2 client drops it, so it only + // reaches the wire over HTTP/1.1, where Firefox sends it. + headers.insert(CONNECTION, HeaderValue::from_static("keep-alive")); + if http1_only { + headers.remove(TE); + } + + let mut orig_headers = OrigHeaderMap::with_capacity(HEADER_ORDER.len()); + for name in HEADER_ORDER { + orig_headers.insert(*name); + } + + let mut builder = wreq::Emulation::builder() + .headers(headers) + .orig_headers(orig_headers); + + if let Some(mut tls) = base.tls_options { + tls.cipher_list = Some(Cow::Borrowed(self.cipher_list)); + tls.curves_list = Some(Cow::Borrowed(self.curves_list)); + builder = builder.tls_options(tls); + } + if let Some(http1) = base.http1_options { + builder = builder.http1_options(http1); + } + if let Some(mut http2) = base.http2_options { + http2.headers_stream_dependency = Some(StreamDependency::new( + StreamId::zero(), + HEADERS_PRIORITY_WEIGHT, + false, + )); + builder = builder.http2_options(http2); + } + + // A group of its own keeps pooled connections from being shared with + // profiles whose TLS handshake differs. + builder.build(Group::new(self.name)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn header<'a>(emulation: &'a wreq::Emulation, name: &str) -> &'a str { + emulation.headers.get(name).unwrap().to_str().unwrap() + } + + #[test] + fn covers_every_release_after_the_bundled_profiles() { + for version in 152..=158 { + let name = format!("firefox_{version}"); + let profile = FirefoxProfile::find(&name).unwrap_or_else(|| panic!("{name} missing")); + assert_eq!(profile.version, format!("{version}.0")); + } + assert!(FirefoxProfile::find("firefox_151").is_none()); + assert!(FirefoxProfile::find("firefox_159").is_none()); + } + + #[test] + fn names_are_unknown_to_wreq_util() { + // A wreq-util profile of the same name takes precedence, so a hit here + // means the local entry is dead: check upstream's profile and drop it. + for profile in PROFILES { + let name = serde_json::Value::String(profile.name.to_owned()); + assert!( + serde_json::from_value::(name).is_err(), + "{} is now bundled with wreq-util", + profile.name + ); + } + } + + #[test] + fn headers_frame_priority_weight_is_corrected() { + let expected = StreamDependency::new(StreamId::zero(), 41, false); + for profile in PROFILES { + let http2 = profile.emulation(Platform::default(), false).http2_options.unwrap(); + assert_eq!(http2.headers_stream_dependency, Some(expected), "{}", profile.name); + } + } + + #[test] + fn user_agent_carries_the_version_on_every_platform() { + let profile = FirefoxProfile::find("firefox_158").unwrap(); + for (platform, expected) in [ + ( + Platform::Windows, + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:158.0) Gecko/20100101 Firefox/158.0", + ), + ( + Platform::MacOS, + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:158.0) Gecko/20100101 Firefox/158.0", + ), + ( + Platform::Linux, + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:158.0) Gecko/20100101 Firefox/158.0", + ), + ( + Platform::Android, + "Mozilla/5.0 (Android 13; Mobile; rv:158.0) Gecko/158.0 Firefox/158.0", + ), + ( + Platform::IOS, + "Mozilla/5.0 (iPhone; CPU iPhone OS 18_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/158.0 Mobile/15E148 Safari/605.1.15", + ), + ] { + let emulation = profile.emulation(platform, false); + assert_eq!(header(&emulation, "user-agent"), expected); + assert_eq!(header(&emulation, "accept-language"), "en-US,en;q=0.9"); + } + } + + fn default_header_names(emulation: &wreq::Emulation) -> Vec { + emulation + .headers + .keys() + .map(|name| name.as_str().to_owned()) + .collect() + } + + #[test] + fn default_headers_are_the_bundled_ones_plus_connection() { + let base = BrowserEmulation::builder() + .profile(BASE_PROFILE) + .build() + .into_emulation(); + let emulation = FirefoxProfile::find("firefox_158") + .unwrap() + .emulation(Platform::default(), false); + + let mut expected = default_header_names(&base); + expected.push("connection".to_owned()); + assert_eq!(default_header_names(&emulation), expected); + assert_eq!(header(&emulation, "connection"), "keep-alive"); + } + + /// Header names of the requests recorded from Firefox 152 and 158 over + /// HTTP/1.1, merged into one sequence, plus `TE`, which only the HTTP/2 + /// recordings carry, and the four names whose place comes from `fetch()` + /// requests and the Firefox sources. Kept apart from `HEADER_ORDER` on + /// purpose, so that an edit there has to be made twice. + const RECORDED_ORDER: [&str; 26] = [ + "Host", "User-Agent", "Accept", "Accept-Language", "Accept-Encoding", "Authorization", + "Range", "Content-Type", "Content-Length", "Origin", "DNT", "Sec-GPC", "Connection", + "Referer", "Cookie", "Upgrade-Insecure-Requests", "Sec-Fetch-Dest", "Sec-Fetch-Mode", + "Sec-Fetch-Site", "Sec-Fetch-User", "If-Modified-Since", "If-None-Match", "Priority", + "Pragma", "Cache-Control", "TE", + ]; + + /// The requests as recorded, each a subsequence of the order. + const RECORDED_REQUESTS: [&[&str]; 5] = [ + // plain navigation + &["Host", "User-Agent", "Accept", "Accept-Language", "Accept-Encoding", "Connection", + "Upgrade-Insecure-Requests", "Sec-Fetch-Dest", "Sec-Fetch-Mode", "Sec-Fetch-Site", + "Sec-Fetch-User", "Priority"], + // repeat navigation with a cookie and cache validators + &["Host", "User-Agent", "Accept", "Accept-Language", "Accept-Encoding", "Connection", + "Cookie", "Upgrade-Insecure-Requests", "Sec-Fetch-Dest", "Sec-Fetch-Mode", + "Sec-Fetch-Site", "Sec-Fetch-User", "If-Modified-Since", "If-None-Match", "Priority"], + // reload + &["Host", "User-Agent", "Accept", "Accept-Language", "Accept-Encoding", "Connection", + "Cookie", "Upgrade-Insecure-Requests", "Sec-Fetch-Dest", "Sec-Fetch-Mode", + "Sec-Fetch-Site", "Priority", "Pragma", "Cache-Control"], + // link with a referrer + &["Host", "User-Agent", "Accept", "Accept-Language", "Accept-Encoding", "Connection", + "Referer", "Cookie", "Upgrade-Insecure-Requests", "Sec-Fetch-Dest", "Sec-Fetch-Mode", + "Sec-Fetch-Site", "Priority"], + // form POST + &["Host", "User-Agent", "Accept", "Accept-Language", "Accept-Encoding", "Content-Type", + "Content-Length", "Origin", "Connection", "Referer", "Cookie", + "Upgrade-Insecure-Requests", "Sec-Fetch-Dest", "Sec-Fetch-Mode", "Sec-Fetch-Site", + "Priority"], + ]; + + #[test] + fn header_order_agrees_with_every_recorded_request() { + for request in RECORDED_REQUESTS { + let positions = request + .iter() + .map(|name| { + HEADER_ORDER + .iter() + .position(|listed| listed == name) + .unwrap_or_else(|| panic!("{name} is missing or spelled differently")) + }) + .collect::>(); + assert!(positions.windows(2).all(|pair| pair[0] < pair[1]), "{request:?}"); + } + } + + #[test] + fn unlisted_headers_go_where_a_script_sets_them() { + let position = |name: &str| { + HEADER_ORDER + .iter() + .position(|listed| listed.eq_ignore_ascii_case(name)) + .unwrap() + }; + let slot = position(UNLISTED_HEADERS_BEFORE.as_str()); + assert!(position("accept-encoding") < slot); + assert!(position("range") < slot); + assert_eq!(HEADER_ORDER[slot], "Content-Type"); + } + + #[test] + fn header_names_are_spelled_and_ordered_like_firefox() { + for profile in PROFILES { + let emulation = profile.emulation(Platform::default(), false); + let spelled = emulation + .orig_headers + .iter() + .map(|(_, name)| String::from_utf8(name.as_ref().to_vec()).unwrap()) + .collect::>(); + assert_eq!(spelled, RECORDED_ORDER, "{}", profile.name); + + // A default header missing from the list would be written in lower + // case after all the others. + for name in emulation.headers.keys() { + assert!( + HEADER_ORDER.iter().any(|listed| listed.eq_ignore_ascii_case(name.as_str())), + "{}: {name} has no place in HEADER_ORDER", + profile.name + ); + } + } + } + + #[test] + fn default_headers_already_follow_the_wire_order() { + // The order list must not reorder the bundled headers, or the HTTP/2 + // request would change as well. + let emulation = FirefoxProfile::find("firefox_158") + .unwrap() + .emulation(Platform::default(), false); + let position = |name: &str| { + HEADER_ORDER + .iter() + .position(|listed| listed.eq_ignore_ascii_case(name)) + .unwrap() + }; + let bundled = default_header_names(&emulation) + .into_iter() + .filter(|name| name != "connection") + .map(|name| position(&name)) + .collect::>(); + assert!(bundled.windows(2).all(|pair| pair[0] < pair[1]), "{bundled:?}"); + } + + #[test] + fn te_is_left_out_only_when_http1_is_certain() { + let profile = FirefoxProfile::find("firefox_158").unwrap(); + let negotiated = profile.emulation(Platform::default(), false); + assert_eq!(header(&negotiated, "te"), "trailers"); + + let http1_only = profile.emulation(Platform::default(), true); + assert!(http1_only.headers.get("te").is_none()); + assert_eq!(header(&http1_only, "connection"), "keep-alive"); + } + + #[test] + fn tls_options_follow_the_release() { + let tls = |name: &str| { + let emulation = FirefoxProfile::find(name) + .unwrap() + .emulation(Platform::default(), false); + let tls = emulation.tls_options.unwrap(); + ( + tls.cipher_list.unwrap().into_owned(), + tls.curves_list.unwrap().into_owned(), + ) + }; + const ECDSA_AES_128_CBC: &str = "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA"; + const ECDSA_AES_256_CBC: &str = "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA"; + + for name in ["firefox_152", "firefox_153"] { + let (ciphers, curves) = tls(name); + assert_eq!(ciphers.split(':').count(), 16, "{name}"); + assert!(ciphers.contains(ECDSA_AES_256_CBC), "{name}"); + assert!(!ciphers.contains(ECDSA_AES_128_CBC), "{name}"); + assert!(curves.ends_with(":ffdhe2048:ffdhe3072"), "{name}"); + } + for name in ["firefox_154", "firefox_155"] { + let (ciphers, curves) = tls(name); + assert_eq!(ciphers.split(':').count(), 15, "{name}"); + assert!(!ciphers.contains(ECDSA_AES_256_CBC), "{name}"); + assert!(curves.ends_with(":ffdhe2048:ffdhe3072"), "{name}"); + } + for name in ["firefox_156", "firefox_157", "firefox_158"] { + let (ciphers, curves) = tls(name); + assert_eq!(ciphers.split(':').count(), 15, "{name}"); + assert!(!ciphers.contains(ECDSA_AES_256_CBC), "{name}"); + assert_eq!(curves, "X25519MLKEM768:X25519:P-256:P-384:P-521", "{name}"); + } + } +} diff --git a/ext/wreq_rb/src/lib.rs b/ext/wreq_rb/src/lib.rs index 880be2b..33c5a4a 100644 --- a/ext/wreq_rb/src/lib.rs +++ b/ext/wreq_rb/src/lib.rs @@ -2,6 +2,7 @@ mod client; mod error; +mod firefox; mod response; use magnus::prelude::*; diff --git a/test/client_test.rb b/test/client_test.rb index c19fd79..44094e5 100644 --- a/test/client_test.rb +++ b/test/client_test.rb @@ -86,6 +86,204 @@ def test_header_order_takes_precedence_over_emulation "got positions #{positions.inspect} in: #{received.inspect}" end + FIREFOX_HTTP1_HEADERS = %w[ + Host User-Agent Accept Accept-Language Accept-Encoding Connection + Upgrade-Insecure-Requests Sec-Fetch-Dest Sec-Fetch-Mode Sec-Fetch-Site + Sec-Fetch-User Priority + ].freeze + + def test_firefox_http1_header_names_and_order + (152..158).each do |version| + client = Wreq::Client.new(emulation: "firefox_#{version}", http1_only: true, no_proxy: true) + received = capture_wire_headers(spelling: true) { |url| client.get(url) } + + assert_equal FIREFOX_HTTP1_HEADERS, received, "firefox_#{version}" + end + end + + def test_firefox_http1_places_request_headers_like_a_navigation + client = Wreq::Client.new(emulation: "firefox_158", http1_only: true, no_proxy: true) + received = capture_wire_headers(spelling: true) do |url| + client.post(url, form: { a: "1" }, headers: { + "cookie" => "sid=1", "referer" => url, "origin" => url.chomp("/"), "x-custom" => "1" + }) + end + + assert_equal %w[ + Host User-Agent Accept Accept-Language Accept-Encoding x-custom Content-Type + Content-Length Origin Connection Referer Cookie Upgrade-Insecure-Requests + Sec-Fetch-Dest Sec-Fetch-Mode Sec-Fetch-Site Sec-Fetch-User Priority + ], received + end + + def test_firefox_http1_places_validators_and_script_headers + client = Wreq::Client.new(emulation: "firefox_158", http1_only: true, no_proxy: true) + received = capture_wire_headers(spelling: true) do |url| + client.get(url, headers: { + "cache-control" => "no-cache", "pragma" => "no-cache", "if-none-match" => '"v1"', + "if-modified-since" => "Mon, 02 Nov 2026 10:00:00 GMT", "range" => "bytes=0-1", + "authorization" => "Bearer t", "sec-gpc" => "1", "dnt" => "1" + }) + end + + assert_equal %w[ + Host User-Agent Accept Accept-Language Accept-Encoding Authorization Range DNT Sec-GPC + Connection Upgrade-Insecure-Requests Sec-Fetch-Dest Sec-Fetch-Mode Sec-Fetch-Site + Sec-Fetch-User If-Modified-Since If-None-Match Priority Pragma Cache-Control + ], received + end + + # Headers the profile has no place for keep the order and the spelling the + # caller gave them, where Firefox writes the headers a script sets. + FIREFOX_HTTP1_WITH_OWN_HEADERS = %w[ + Host User-Agent Accept Accept-Language Accept-Encoding X-Requested-With X-CSRF-Token x-c + x-b x-a Connection Upgrade-Insecure-Requests Sec-Fetch-Dest Sec-Fetch-Mode Sec-Fetch-Site + Sec-Fetch-User Priority + ].freeze + OWN_HEADERS = { + "X-Requested-With" => "XMLHttpRequest", "X-CSRF-Token" => "t", "x-c" => "3", "x-b" => "2", "x-a" => "1" + }.freeze + + def test_firefox_keeps_the_order_of_request_headers_it_does_not_know + client = Wreq::Client.new(emulation: "firefox_158", http1_only: true, no_proxy: true) + received = capture_wire_headers(spelling: true) { |url| client.get(url, headers: OWN_HEADERS) } + + assert_equal FIREFOX_HTTP1_WITH_OWN_HEADERS, received + end + + def test_firefox_keeps_the_order_of_client_headers_it_does_not_know + client = Wreq::Client.new( + emulation: "firefox_158", http1_only: true, no_proxy: true, headers: OWN_HEADERS.first(2).to_h + ) + received = capture_wire_headers(spelling: true) do |url| + client.get(url, headers: OWN_HEADERS.drop(2).to_h) + end + + assert_equal FIREFOX_HTTP1_WITH_OWN_HEADERS, received + end + + def test_firefox_per_request_emulation_keeps_the_order_of_unknown_headers + client = Wreq::Client.new( + emulation: false, http1_only: true, no_proxy: true, headers: OWN_HEADERS.first(2).to_h + ) + received = capture_wire_headers(spelling: true) do |url| + client.get(url, emulation: "firefox_158", headers: OWN_HEADERS.drop(2).to_h) + end + + assert_equal FIREFOX_HTTP1_WITH_OWN_HEADERS, received + end + + def test_firefox_batch_requests_keep_the_order_of_unknown_headers + client = Wreq::Client.new(emulation: "firefox_158", http1_only: true, no_proxy: true) + received = capture_wire_headers(spelling: true) do |url| + client.request_batch( + [{ url: url, headers: OWN_HEADERS.drop(2).to_h }], headers: OWN_HEADERS.first(2).to_h + ) + end + + assert_equal FIREFOX_HTTP1_WITH_OWN_HEADERS, received + end + + def test_firefox_connection_header_can_be_overridden + client = Wreq::Client.new(emulation: "firefox_158", http1_only: true, no_proxy: true) + received = capture_wire_headers(lines: true) do |url| + client.get(url, headers: { "connection" => "close" }) + end + + assert_equal FIREFOX_HTTP1_HEADERS, received.map { |line| line.split(":", 2).first } + assert_equal ["Connection: close"], received.grep(/\Aconnection:/i) + end + + def test_firefox_places_client_headers_without_request_headers + client = Wreq::Client.new( + emulation: "firefox_158", http1_only: true, no_proxy: true, headers: OWN_HEADERS + ) + received = capture_wire_headers(spelling: true) { |url| client.get(url) } + + assert_equal FIREFOX_HTTP1_WITH_OWN_HEADERS, received + end + + def test_firefox_per_request_emulation_places_client_headers + # No header on the request itself, or only one the client already sends. + [{}, { headers: { "x-b" => "9" } }].each do |options| + [false, "firefox_157"].each do |client_emulation| + client = Wreq::Client.new( + emulation: client_emulation, http1_only: true, no_proxy: true, headers: OWN_HEADERS + ) + received = capture_wire_headers(spelling: true) do |url| + client.get(url, emulation: "firefox_158", **options) + end + + assert_equal FIREFOX_HTTP1_WITH_OWN_HEADERS, received, "#{client_emulation.inspect} #{options}" + end + end + end + + def test_firefox_sends_te_unless_http1_is_forced + # Firefox sends TE over HTTP/2 only. The header is a default one, which + # cannot depend on the protocol of a connection, so a client that is not + # restricted to HTTP/1.1 sends it over HTTP/1.1 as well, as it does here. + client = Wreq::Client.new(emulation: "firefox_158", no_proxy: true) + received = capture_wire_headers(spelling: true) { |url| client.get(url) } + + assert_equal FIREFOX_HTTP1_HEADERS + ["TE"], received + end + + def test_firefox_per_request_emulation_uses_firefox_header_names + # The client has no emulation of its own: its default headers would be + # sent along with the ones of the per-request profile. + client = Wreq::Client.new(emulation: false, http1_only: true, no_proxy: true) + received = capture_wire_headers(spelling: true) { |url| client.get(url, emulation: "firefox_158") } + + assert_equal FIREFOX_HTTP1_HEADERS, received + end + + def test_header_order_takes_precedence_over_firefox_emulation + # The user's names keep their position and their spelling; the profile + # orders and spells only the headers the user did not list. + order = ["host", "accept", "USER-AGENT"] + client = Wreq::Client.new( + emulation: "firefox_158", + http1_only: true, + no_proxy: true, + header_order: order + ) + received = capture_wire_headers(spelling: true) { |url| client.get(url) } + + assert_equal order + (FIREFOX_HTTP1_HEADERS - %w[Host Accept User-Agent]), received + end + + def test_header_order_with_firefox_places_unknown_headers + client = Wreq::Client.new( + emulation: "firefox_158", + http1_only: true, + no_proxy: true, + header_order: ["host", "x-b"] + ) + received = capture_wire_headers(spelling: true) do |url| + client.get(url, headers: { "X-A" => "1", "X-B" => "2", "X-C" => "3" }) + end + + assert_equal %w[host x-b User-Agent Accept Accept-Language Accept-Encoding X-A X-C] + + FIREFOX_HTTP1_HEADERS.drop(5), received + end + + def test_header_order_naming_content_type_keeps_unknown_headers_before_the_rest + client = Wreq::Client.new( + emulation: "firefox_158", + http1_only: true, + no_proxy: true, + header_order: ["Content-Type"] + ) + received = capture_wire_headers(spelling: true) do |url| + client.post(url, json: { a: 1 }, headers: { "X-A" => "1", "X-B" => "2" }) + end + + assert_equal %w[ + Content-Type Host User-Agent Accept Accept-Language Accept-Encoding X-A X-B Content-Length + ] + FIREFOX_HTTP1_HEADERS.drop(5), received + end + def test_connect_timeout_defaults_to_client_timeout [{ timeout: 0.25 }, { timeout: 0.25, connect_timeout: nil }].each do |options| elapsed = stalled_tls_request_duration(options) @@ -142,7 +340,10 @@ def stalled_tls_request_duration(options, request_timeout: 3) # Spins up a local TCP server, yields the port formatted into a URL, captures # the header names from the raw HTTP/1.1 request, then tears down the server. - def capture_wire_headers + # Header names of the request the block sends, in wire order. They are + # lower-cased unless +spelling+ is true; +lines+ returns the whole header + # lines instead. + def capture_wire_headers(spelling: false, lines: false) require "socket" server = TCPServer.new("127.0.0.1", 0) port = server.addr[1] @@ -153,7 +354,8 @@ def capture_wire_headers loop do line = conn.gets&.chomp break if line.nil? || line.empty? - received << line.split(":", 2).first.downcase + name = line.split(":", 2).first + received << (lines ? line : (spelling ? name : name.downcase)) end conn.write "HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" conn.close diff --git a/test/emulation_test.rb b/test/emulation_test.rb index f9a6082..4fc09d9 100644 --- a/test/emulation_test.rb +++ b/test/emulation_test.rb @@ -16,6 +16,27 @@ def test_explicit_emulation_firefox assert_match(/Firefox/, body["headers"]["User-Agent"]) end + def test_emulation_firefox_releases_newer_than_wreq_util + (152..158).each do |version| + client = Wreq::Client.new(emulation: "firefox_#{version}") + headers = client.get("https://httpbun.com/headers").json["headers"] + assert_match(%r{rv:#{version}\.0\) Gecko/20100101 Firefox/#{version}\.0\z}, headers["User-Agent"]) + assert_equal "en-US,en;q=0.9", headers["Accept-Language"] + end + end + + def test_emulation_firefox_158_per_request_and_os + client = Wreq::Client.new(emulation: false) + resp = client.get("https://httpbun.com/headers", emulation: "firefox_158", emulation_os: "windows") + assert_equal "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:158.0) Gecko/20100101 Firefox/158.0", + resp.json["headers"]["User-Agent"] + end + + def test_emulation_unknown_firefox_release + error = assert_raises(Wreq::Error) { Wreq::Client.new(emulation: "firefox_999") } + assert_match(/unknown emulation: 'firefox_999'/, error.message) + end + def test_emulation_disabled client = Wreq::Client.new(emulation: false, user_agent: "custom-agent/1.0") resp = client.get("https://httpbun.com/headers")