From 304edf9e25de59f0ba8ca6e54d82e987dd3d4234 Mon Sep 17 00:00:00 2001 From: Albert Hui Date: Thu, 24 Sep 2026 21:30:37 +0800 Subject: [PATCH 1/2] test(interpret): specify Google ei microseconds, caveat and param match (RED) The ei= decoder reads only the 4-byte little-endian seconds. The bytes after them are protobuf varints, the first a microsecond count: in the unfurl issue #56 URL, ei decodes to 1587403446 s + 540099 us, and the ved parameter in the same URL carries 1587403446540099 us (protobuf 13.1.1), which unfurl also reports. New tests pin: - the microsecond varint (unfurl #56 and the Cheeky4n6Monkey/Deed Poll example), plus an env-gated unfurl oracle to the microsecond; - a whole-seconds note when the varint is truncated or >= 1e6; - the caveat that ei is the page-serve time (session start or previous search), not necessarily the query time (unfurl #56); - matching only the ei / sei parameter names, not gei= / rei=. Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/catalog.rs | 71 ++++++++++++++++++++++++++++++++++++++++++ tests/unfurl_oracle.rs | 21 +++++++++++++ 2 files changed, 92 insertions(+) diff --git a/tests/catalog.rs b/tests/catalog.rs index 7d5f0b1..74b66d2 100644 --- a/tests/catalog.rs +++ b/tests/catalog.rs @@ -148,6 +148,77 @@ fn google_ei_url_parameter_first_4_bytes_le_unix_seconds() { ); } +/// The `google_ei` reading of `input`: its rendered instant and assumption text. +fn ei_reading(input: &str) -> (String, String) { + let c = interpret::interpret_string(input) + .into_iter() + .find(|c| c.format_id == "google_ei") + .unwrap_or_else(|| panic!("no google_ei candidate for {input:?}")); + (c.rendered.unwrap_or_default(), c.assumptions.join(" ")) +} + +#[test] +fn google_ei_decodes_the_microsecond_varint_after_the_seconds() { + // Real URL from unfurl issue #56 (Rasmus-Riis, 2020). After the 4-byte LE + // seconds comes a protobuf varint of microseconds (540099). Corroborated by a + // separate Google field in the SAME URL: `ved` protobuf 13→1→1 carries + // 1587403446540099 µs, and unfurl renders the same µs value. + let (r, note) = + ei_reading("https://www.google.com/search?ei=ttqdXsP7IMKZk74Pgv-k6AY&q=third+search"); + assert_eq!(r, "2020-04-20T17:24:06.540099Z"); + assert!(note.contains("microsecond"), "{note}"); + // Cheeky4n6Monkey / Deed Poll Office example (seconds 1387841717 published + // there; the µs varint is 616780). + let (r, _) = ei_reading("ei=tci4UszSJeLN7Ab9xYD4CQ"); + assert_eq!(r, "2013-12-23T23:35:17.616780Z"); +} + +#[test] +fn google_ei_says_it_is_the_page_serve_time_not_the_query_time() { + // unfurl #56: ei was minted when Google served the page the user searched + // FROM (session start / previous search), minutes to hours before the query + // in the same URL. The reading must carry that caveat, not imply search time. + let (_, note) = ei_reading("ei=ttqdXsP7IMKZk74Pgv-k6AY"); + assert!(note.contains("not necessarily"), "{note}"); +} + +#[test] +fn google_ei_without_a_usable_microsecond_field_says_so() { + // Synthetic (python: urlsafe_b64encode(pack(' Date: Thu, 24 Sep 2026 21:35:33 +0800 Subject: [PATCH 2/2] feat(interpret): decode Google ei microseconds, and caveat its meaning (GREEN) Google's ei= parameter is 4 bytes of little-endian Unix seconds followed by protobuf varints; the first is a microsecond count. The decoder now reads it, so the unfurl #56 URL renders 2020-04-20T17:24:06.540099Z rather than truncating to the second. This matches unfurl and the ved parameter carried in the same URL. - When the varint is missing, unterminated, or >= 1 000 000, the reading falls back to whole seconds and its note says so, so a truncated value is not presented as second-exact. - Every reading notes that ei is when Google served the page the link was minted on (session start or a previous search), not necessarily when the query in the URL ran (unfurl #56). - Only the ei and sei parameter names match. The old split("ei=") also fired on gei=, rei= and any other name ending in ei. - google_ei moves from the fixed-note STRING_FORMATS table to a push_google_ei helper (like push_jwt) because its note now varies. - docs/formats/identifiers.md gains a Google ei section with sources. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/formats/identifiers.md | 33 ++++++++++++- src/interpret.rs | 95 +++++++++++++++++++++++++++---------- tests/catalog.rs | 10 ++-- 3 files changed, 108 insertions(+), 30 deletions(-) diff --git a/docs/formats/identifiers.md b/docs/formats/identifiers.md index 31dff4a..1600829 100644 --- a/docs/formats/identifiers.md +++ b/docs/formats/identifiers.md @@ -2,8 +2,9 @@ title: "Identifiers with embedded timestamps — Snowflake, UUID, ULID, ObjectId" description: >- Forensic reference for IDs that embed a creation time: Twitter/X and Discord - Snowflakes, UUID v1/v6/v7 (RFC 9562), ULID, MongoDB ObjectId, KSUID, and - Sonyflake — with bit layouts, epochs, extraction formulas, and worked examples. + Snowflakes, UUID v1/v6/v7 (RFC 9562), ULID, MongoDB ObjectId, KSUID, + Sonyflake, and Google's ei= search parameter — with bit layouts, epochs, + extraction formulas, and worked examples. --- # Identifiers with embedded timestamps @@ -101,6 +102,33 @@ default epoch **2014-09-01** (`1409529600000` ms). Source: of the host's private IPv4 address (partial host leak). Epoch, time unit, and bit split are configurable — confirm against the generating code. +## Google `ei=` search parameter {#google-ei} + +Google search URLs carry an `ei` (and sometimes `sei`) parameter: unpadded urlsafe +base64 whose first **4 bytes are little-endian Unix seconds**, followed by protobuf +varints — the first a **microsecond** count. Sources: +[Cheeky4n6Monkey, “Google-ei’d ?!” (2014)](https://cheeky4n6monkey.blogspot.com/2014/10/google-eid.html), +[Kevin Jones, Deed Poll Office (2013)](https://deedpolloffice.com/blog/articles/decoding-ei-parameter), +[unfurl `parse_google.py`](https://github.com/obsidianforensics/unfurl/blob/main/unfurl/parsers/parse_google.py). +Google does not document the layout; it is reverse-engineered. + +```text +$ timeglyph 'https://www.google.com/search?ei=ttqdXsP7IMKZk74Pgv-k6AY&q=x' + [1.00] google_ei 2020-04-20T17:24:06.540099Z +``` + +- **Microseconds:** in that URL (from [unfurl #56](https://github.com/obsidianforensics/unfurl/issues/56)), + the `ved` parameter carries `1587403446540099` µs in its protobuf field 13→1→1 — + the same instant as ei's seconds + microsecond varint. unfurl reports the same value. + If the varint is missing or not below 1 000 000, timeglyph reports whole seconds and + says so in the reading. +- **Gotcha:** ei is when Google **served the page the link was minted on** (session + start or a previous search), not necessarily when the query in the same URL was run. + The unfurl #56 reporter saw ei values “hours apart from the actual search”. +- **Recognised only as a named parameter** (`ei=` / `sei=`, at the start or after + `?`, `&`, `#`). A bare token has no structure to detect it by, so pass it as + `ei=`. + ## Cross-scheme summary | Scheme | TS bits | Resolution | Epoch (UTC) | Extraction core | @@ -113,6 +141,7 @@ default epoch **2014-09-01** (`1409529600000` ms). Source: | MongoDB ObjectId | 32 | 1 s | 1970-01-01 | first 4 bytes (BE) | | KSUID | 32 | 1 s | 2014-05-13 | `BE4bytes + 1400000000` | | Sonyflake | 39 | 10 ms | 2014-09-01 | `(id>>24)*10 + epoch` | +| Google `ei=` | 32 + varint | 1 µs | 1970-01-01 | first 4 bytes (LE) + µs varint | **Highest attribution value:** UUIDv1/v6 `node` (often real MAC); Sonyflake machine id (private IP low bits); ObjectId per-process random (legacy: machine-id + PID). diff --git a/src/interpret.rs b/src/interpret.rs index bc27f4f..e35950b 100644 --- a/src/interpret.rs +++ b/src/interpret.rs @@ -968,13 +968,6 @@ const STRING_FORMATS: &[StringFormat] = &[ spec: "MongoDB ObjectId spec (4-byte big-endian Unix-seconds prefix)", note: "parsed as a MongoDB ObjectId — the first 4 bytes are big-endian Unix seconds", }, - StringFormat { - parse: parse_google_ei, - id: "google_ei", - label: "Google ei= URL parameter (Unix seconds in the first 4 bytes)", - spec: "Google ei URL param (urlsafe base64; first 4 bytes little-endian Unix seconds)", - note: "parsed as a Google ei= URL parameter — the leading 4 bytes are little-endian Unix seconds", - }, StringFormat { parse: parse_clf, id: "clf", @@ -1031,10 +1024,12 @@ const STRING_FORMATS: &[StringFormat] = &[ pub fn interpret_string(text: &str) -> Vec { let s = text.trim(); let mut out = Vec::new(); - // Dynamic-note formats first (ISO 8601 + ASN.1 + JWT), then the fixed-note registry. + // Dynamic-note formats first (ISO 8601 + ASN.1 + JWT + Google ei), then the + // fixed-note registry. push_iso8601(s, &mut out); push_asn1(s, &mut out); push_jwt(s, &mut out); + push_google_ei(s, &mut out); for f in STRING_FORMATS { if let Some(instant) = (f.parse)(s) { out.push(string_candidate(f.id, f.label, f.spec, instant, f.note)); @@ -1278,23 +1273,73 @@ fn parse_objectid(s: &str) -> Option { Some(PosixNs(secs.checked_mul(Unit::Seconds.nanos())?)) } -/// Google's `ei=` URL parameter (urlsafe base64): its leading 4 decoded bytes are -/// a little-endian Unix-seconds count. Decoded ONLY when the `ei=` marker is -/// present (the format *is* a named URL parameter) — a bare base64-looking token -/// carries no structural signature, so requiring the marker keeps auto-detect -/// quiet instead of reading a timestamp out of any 6-char word. `None` if no -/// `ei=` marker, or the value is under 6 chars / not urlsafe base64. -fn parse_google_ei(s: &str) -> Option { - // The value after the `ei=` marker, up to the next query delimiter. - let val = s.split("ei=").nth(1)?.split(['&', '#']).next()?; - // 6 urlsafe-base64 chars = 36 bits; the first 4 bytes are the top 32. - let mut acc: u64 = 0; - for ch in val.get(..6)?.bytes() { - acc = (acc << 6) | u64::from(urlsafe_b64_val(ch)?); - } - let bytes = ((acc >> 4) as u32).to_be_bytes(); - let secs = i128::from(u32::from_le_bytes(bytes)); - Some(PosixNs(secs.checked_mul(Unit::Seconds.nanos())?)) +/// Why an `ei` reading is only as precise as it is: the instant is when Google +/// served the page the link was minted on, which unfurl issue #56 showed can be +/// hours before the query carried in the same URL. +const EI_SERVE_TIME: &str = "this is when Google served the page the link was minted on \ +(session start or a previous search), not necessarily when the query in the same URL was run"; + +/// Google's `ei=` (and `sei=`) URL parameter: unpadded urlsafe base64 whose +/// leading 4 bytes are little-endian Unix seconds, followed by protobuf varints — +/// the first a microsecond count (it reappears as `ved` protobuf 13→1→1 in the +/// same URL; unfurl renders it the same way). Decoded ONLY as a named query +/// parameter: a bare base64-looking token carries no structural signature, so +/// requiring the name keeps auto-detect quiet instead of reading a timestamp out +/// of any 6-char word. When the microsecond varint is missing or out of range the +/// reading is whole seconds and its note says so. +fn push_google_ei(s: &str, out: &mut Vec) { + let Some(val) = s + .split(['?', '&', '#']) + .find_map(|param| match param.split_once('=') { + Some(("ei" | "sei", v)) => Some(v), + _ => None, + }) + else { + return; + }; + let Some(bytes) = b64url_decode(val) else { + return; + }; + let Some((secs, rest)) = bytes.split_first_chunk::<4>() else { + return; + }; + let secs = i128::from(u32::from_le_bytes(*secs)); + let micros = ei_micros(rest); + let instant = PosixNs(secs * Unit::Seconds.nanos() + i128::from(micros.unwrap_or(0)) * 1_000); + let note = if micros.is_some() { + format!( + "parsed as a Google ei= URL parameter — 4 bytes little-endian Unix seconds + a \ + varint of microseconds; {EI_SERVE_TIME}" + ) + } else { + format!( + "parsed as a Google ei= URL parameter — whole seconds only: no microsecond varint \ + in 0–999999 follows the 4 little-endian Unix-seconds bytes (value truncated?); \ + {EI_SERVE_TIME}" + ) + }; + out.push(string_candidate( + "google_ei", + "Google ei= URL parameter (Unix seconds + microseconds)", + "Google ei URL param (urlsafe base64; 4-byte LE Unix seconds, varint µs) — \ + Cheeky4n6Monkey 2014; unfurl", + instant, + ¬e, + )); +} + +/// The microsecond varint leading `bytes` (protobuf base-128), or `None` if it +/// does not terminate or is not below 1 000 000. Any value in range fits in 3 +/// varint bytes (2^21 > 10^6), so a longer varint is out of range by construction. +fn ei_micros(bytes: &[u8]) -> Option { + let mut v: u32 = 0; + for (i, &b) in bytes.iter().take(3).enumerate() { + v |= u32::from(b & 0x7F) << (7 * i); + if b & 0x80 == 0 { + return (v < 1_000_000).then_some(v); + } + } + None } /// One urlsafe-base64 character (`A–Z a–z 0–9 - _`) to its 6-bit value; `None` diff --git a/tests/catalog.rs b/tests/catalog.rs index 74b66d2..79af6a6 100644 --- a/tests/catalog.rs +++ b/tests/catalog.rs @@ -170,13 +170,14 @@ fn google_ei_decodes_the_microsecond_varint_after_the_seconds() { // Cheeky4n6Monkey / Deed Poll Office example (seconds 1387841717 published // there; the µs varint is 616780). let (r, _) = ei_reading("ei=tci4UszSJeLN7Ab9xYD4CQ"); - assert_eq!(r, "2013-12-23T23:35:17.616780Z"); + // (the renderer drops trailing zeros: .616780 → .61678) + assert_eq!(r, "2013-12-23T23:35:17.61678Z"); } #[test] fn google_ei_says_it_is_the_page_serve_time_not_the_query_time() { // unfurl #56: ei was minted when Google served the page the user searched - // FROM (session start / previous search), minutes to hours before the query + // FROM (session start / previous search), up to hours before the query // in the same URL. The reading must carry that caveat, not imply search time. let (_, note) = ei_reading("ei=ttqdXsP7IMKZk74Pgv-k6AY"); assert!(note.contains("not necessarily"), "{note}"); @@ -205,10 +206,13 @@ fn google_ei_matches_only_the_ei_and_sei_parameter_names() { let (r, _) = ei_reading("https://www.google.com.au/search?q=bananas&gbv=1&sei=BrU2VKfrB9Xz8gX2iILoBA"); assert!(r.starts_with("2014-10-09T16:17:10"), "{r}"); - // A parameter merely ENDING in "ei" is a different parameter. + // A parameter merely ENDING in "ei" is a different parameter; a value that is + // not urlsafe base64, or decodes to under the 4 seconds bytes, is no reading. for other in [ "?gei=ttqdXsP7IMKZk74Pgv-k6AY", "x?q=1&rei=ttqdXsP7IMKZk74Pgv-k6AY", + "ei=ttqd+sP7", + "ei=ttqd", ] { assert!( !interpret::interpret_string(other)