feat(interpret): decode Google ei microseconds, and caveat its meaning

docs/formats/identifiers.md

@@ -2,8 +2,9 @@
 title: "Identifiers with embedded timestamps — Snowflake, UUID, ULID, ObjectId"
 description: >-
   Forensic reference for IDs that embed a creation time: Twitter/X and Discord
-  Snowflakes, UUID v1/v6/v7 (RFC 9562), ULID, MongoDB ObjectId, KSUID, and
-  Sonyflake — with bit layouts, epochs, extraction formulas, and worked examples.
+  Snowflakes, UUID v1/v6/v7 (RFC 9562), ULID, MongoDB ObjectId, KSUID,
+  Sonyflake, and Google's ei= search parameter — with bit layouts, epochs,
+  extraction formulas, and worked examples.
 ---
 
 # Identifiers with embedded timestamps
@@ -101,6 +102,33 @@ default epoch **2014-09-01** (`1409529600000` ms). Source:
   of the host's private IPv4 address (partial host leak). Epoch, time unit, and bit split
   are configurable — confirm against the generating code.
 
+## Google `ei=` search parameter {#google-ei}
+
+Google search URLs carry an `ei` (and sometimes `sei`) parameter: unpadded urlsafe
+base64 whose first **4 bytes are little-endian Unix seconds**, followed by protobuf
+varints — the first a **microsecond** count. Sources:
+[Cheeky4n6Monkey, “Google-ei’d ?!” (2014)](https://cheeky4n6monkey.blogspot.com/2014/10/google-eid.html),
+[Kevin Jones, Deed Poll Office (2013)](https://deedpolloffice.com/blog/articles/decoding-ei-parameter),
+[unfurl `parse_google.py`](https://github.com/obsidianforensics/unfurl/blob/main/unfurl/parsers/parse_google.py).
+Google does not document the layout; it is reverse-engineered.
+
+```text
+$ timeglyph 'https://www.google.com/search?ei=ttqdXsP7IMKZk74Pgv-k6AY&q=x'
+  [1.00] google_ei        2020-04-20T17:24:06.540099Z
+```
+
+- **Microseconds:** in that URL (from [unfurl #56](https://github.com/obsidianforensics/unfurl/issues/56)),
+  the `ved` parameter carries `1587403446540099` µs in its protobuf field 13→1→1 —
+  the same instant as ei's seconds + microsecond varint. unfurl reports the same value.
+  If the varint is missing or not below 1 000 000, timeglyph reports whole seconds and
+  says so in the reading.
+- **Gotcha:** ei is when Google **served the page the link was minted on** (session
+  start or a previous search), not necessarily when the query in the same URL was run.
+  The unfurl #56 reporter saw ei values “hours apart from the actual search”.
+- **Recognised only as a named parameter** (`ei=` / `sei=`, at the start or after
+  `?`, `&`, `#`). A bare token has no structure to detect it by, so pass it as
+  `ei=<value>`.
+
 ## Cross-scheme summary
 
 | Scheme | TS bits | Resolution | Epoch (UTC) | Extraction core |
@@ -113,6 +141,7 @@ default epoch **2014-09-01** (`1409529600000` ms). Source:
 | MongoDB ObjectId | 32 | 1 s | 1970-01-01 | first 4 bytes (BE) |
 | KSUID | 32 | 1 s | 2014-05-13 | `BE4bytes + 1400000000` |
 | Sonyflake | 39 | 10 ms | 2014-09-01 | `(id>>24)*10 + epoch` |
+| Google `ei=` | 32 + varint | 1 µs | 1970-01-01 | first 4 bytes (LE) + µs varint |
 
 **Highest attribution value:** UUIDv1/v6 `node` (often real MAC); Sonyflake machine id
 (private IP low bits); ObjectId per-process random (legacy: machine-id + PID).

src/interpret.rs

@@ -968,13 +968,6 @@ const STRING_FORMATS: &[StringFormat] = &[
         spec: "MongoDB ObjectId spec (4-byte big-endian Unix-seconds prefix)",
         note: "parsed as a MongoDB ObjectId — the first 4 bytes are big-endian Unix seconds",
     },
-    StringFormat {
-        parse: parse_google_ei,
-        id: "google_ei",
-        label: "Google ei= URL parameter (Unix seconds in the first 4 bytes)",
-        spec: "Google ei URL param (urlsafe base64; first 4 bytes little-endian Unix seconds)",
-        note: "parsed as a Google ei= URL parameter — the leading 4 bytes are little-endian Unix seconds",
-    },
     StringFormat {
         parse: parse_clf,
         id: "clf",
@@ -1031,10 +1024,12 @@ const STRING_FORMATS: &[StringFormat] = &[
 pub fn interpret_string(text: &str) -> Vec<Candidate> {
     let s = text.trim();
     let mut out = Vec::new();
-    // Dynamic-note formats first (ISO 8601 + ASN.1 + JWT), then the fixed-note registry.
+    // Dynamic-note formats first (ISO 8601 + ASN.1 + JWT + Google ei), then the
+    // fixed-note registry.
     push_iso8601(s, &mut out);
     push_asn1(s, &mut out);
     push_jwt(s, &mut out);
+    push_google_ei(s, &mut out);
     for f in STRING_FORMATS {
         if let Some(instant) = (f.parse)(s) {
             out.push(string_candidate(f.id, f.label, f.spec, instant, f.note));
@@ -1278,23 +1273,73 @@ fn parse_objectid(s: &str) -> Option<PosixNs> {
     Some(PosixNs(secs.checked_mul(Unit::Seconds.nanos())?))
 }
 
-/// Google's `ei=` URL parameter (urlsafe base64): its leading 4 decoded bytes are
-/// a little-endian Unix-seconds count. Decoded ONLY when the `ei=` marker is
-/// present (the format *is* a named URL parameter) — a bare base64-looking token
-/// carries no structural signature, so requiring the marker keeps auto-detect
-/// quiet instead of reading a timestamp out of any 6-char word. `None` if no
-/// `ei=` marker, or the value is under 6 chars / not urlsafe base64.
-fn parse_google_ei(s: &str) -> Option<PosixNs> {
-    // The value after the `ei=` marker, up to the next query delimiter.
-    let val = s.split("ei=").nth(1)?.split(['&', '#']).next()?;
-    // 6 urlsafe-base64 chars = 36 bits; the first 4 bytes are the top 32.
-    let mut acc: u64 = 0;
-    for ch in val.get(..6)?.bytes() {
-        acc = (acc << 6) | u64::from(urlsafe_b64_val(ch)?);
-    }
-    let bytes = ((acc >> 4) as u32).to_be_bytes();
-    let secs = i128::from(u32::from_le_bytes(bytes));
-    Some(PosixNs(secs.checked_mul(Unit::Seconds.nanos())?))
+/// Why an `ei` reading is only as precise as it is: the instant is when Google
+/// served the page the link was minted on, which unfurl issue #56 showed can be
+/// hours before the query carried in the same URL.
+const EI_SERVE_TIME: &str = "this is when Google served the page the link was minted on \
+(session start or a previous search), not necessarily when the query in the same URL was run";
+
+/// Google's `ei=` (and `sei=`) URL parameter: unpadded urlsafe base64 whose
+/// leading 4 bytes are little-endian Unix seconds, followed by protobuf varints —
+/// the first a microsecond count (it reappears as `ved` protobuf 13→1→1 in the
+/// same URL; unfurl renders it the same way). Decoded ONLY as a named query
+/// parameter: a bare base64-looking token carries no structural signature, so
+/// requiring the name keeps auto-detect quiet instead of reading a timestamp out
+/// of any 6-char word. When the microsecond varint is missing or out of range the
+/// reading is whole seconds and its note says so.
+fn push_google_ei(s: &str, out: &mut Vec<Candidate>) {
+    let Some(val) = s
+        .split(['?', '&', '#'])
+        .find_map(|param| match param.split_once('=') {
+            Some(("ei" | "sei", v)) => Some(v),
+            _ => None,
+        })
+    else {
+        return;
+    };
+    let Some(bytes) = b64url_decode(val) else {
+        return;
+    };
+    let Some((secs, rest)) = bytes.split_first_chunk::<4>() else {
+        return;
+    };
+    let secs = i128::from(u32::from_le_bytes(*secs));
+    let micros = ei_micros(rest);
+    let instant = PosixNs(secs * Unit::Seconds.nanos() + i128::from(micros.unwrap_or(0)) * 1_000);
+    let note = if micros.is_some() {
+        format!(
+            "parsed as a Google ei= URL parameter — 4 bytes little-endian Unix seconds + a \
+             varint of microseconds; {EI_SERVE_TIME}"
+        )
+    } else {
+        format!(
+            "parsed as a Google ei= URL parameter — whole seconds only: no microsecond varint \
+             in 0–999999 follows the 4 little-endian Unix-seconds bytes (value truncated?); \
+             {EI_SERVE_TIME}"
+        )
+    };
+    out.push(string_candidate(
+        "google_ei",
+        "Google ei= URL parameter (Unix seconds + microseconds)",
+        "Google ei URL param (urlsafe base64; 4-byte LE Unix seconds, varint µs) — \
+         Cheeky4n6Monkey 2014; unfurl",
+        instant,
+        &note,
+    ));
+}
+
+/// The microsecond varint leading `bytes` (protobuf base-128), or `None` if it
+/// does not terminate or is not below 1 000 000. Any value in range fits in 3
+/// varint bytes (2^21 > 10^6), so a longer varint is out of range by construction.
+fn ei_micros(bytes: &[u8]) -> Option<u32> {
+    let mut v: u32 = 0;
+    for (i, &b) in bytes.iter().take(3).enumerate() {
+        v |= u32::from(b & 0x7F) << (7 * i);
+        if b & 0x80 == 0 {
+            return (v < 1_000_000).then_some(v);
+        }
+    }
+    None
 }
 
 /// One urlsafe-base64 character (`A–Z a–z 0–9 - _`) to its 6-bit value; `None`

tests/catalog.rs

@@ -148,6 +148,81 @@ fn google_ei_url_parameter_first_4_bytes_le_unix_seconds() {
     );
 }
 
+/// The `google_ei` reading of `input`: its rendered instant and assumption text.
+fn ei_reading(input: &str) -> (String, String) {
+    let c = interpret::interpret_string(input)
+        .into_iter()
+        .find(|c| c.format_id == "google_ei")
+        .unwrap_or_else(|| panic!("no google_ei candidate for {input:?}"));
+    (c.rendered.unwrap_or_default(), c.assumptions.join(" "))
+}
+
+#[test]
+fn google_ei_decodes_the_microsecond_varint_after_the_seconds() {
+    // Real URL from unfurl issue #56 (Rasmus-Riis, 2020). After the 4-byte LE
+    // seconds comes a protobuf varint of microseconds (540099). Corroborated by a
+    // separate Google field in the SAME URL: `ved` protobuf 13→1→1 carries
+    // 1587403446540099 µs, and unfurl renders the same µs value.
+    let (r, note) =
+        ei_reading("https://www.google.com/search?ei=ttqdXsP7IMKZk74Pgv-k6AY&q=third+search");
+    assert_eq!(r, "2020-04-20T17:24:06.540099Z");
+    assert!(note.contains("microsecond"), "{note}");
+    // Cheeky4n6Monkey / Deed Poll Office example (seconds 1387841717 published
+    // there; the µs varint is 616780).
+    let (r, _) = ei_reading("ei=tci4UszSJeLN7Ab9xYD4CQ");
+    // (the renderer drops trailing zeros: .616780 → .61678)
+    assert_eq!(r, "2013-12-23T23:35:17.61678Z");
+}
+
+#[test]
+fn google_ei_says_it_is_the_page_serve_time_not_the_query_time() {
+    // unfurl #56: ei was minted when Google served the page the user searched
+    // FROM (session start / previous search), up to hours before the query
+    // in the same URL. The reading must carry that caveat, not imply search time.
+    let (_, note) = ei_reading("ei=ttqdXsP7IMKZk74Pgv-k6AY");
+    assert!(note.contains("not necessarily"), "{note}");
+}
+
+#[test]
+fn google_ei_without_a_usable_microsecond_field_says_so() {
+    // Synthetic (python: urlsafe_b64encode(pack('<I',1587403446)+tail)):
+    // tail 0xC3 (varint never terminates) and tail C0 84 3D (= 1_000_000, not a
+    // microsecond count) — both fall back to whole seconds and SAY so, rather
+    // than inventing a fraction or rendering a bare second as if it were exact.
+    for tok in ["ttqdXsM", "ttqdXsCEPQ", "Yx1sYw"] {
+        let (r, note) = ei_reading(&format!("ei={tok}"));
+        assert!(r.ends_with(":06Z") || tok == "Yx1sYw", "{tok}: {r}");
+        assert!(note.contains("whole seconds"), "{tok}: {note}");
+    }
+    // 999_999 is the largest valid microsecond value.
+    let (r, _) = ei_reading("ei=ttqdXr-EPQ");
+    assert_eq!(r, "2020-04-20T17:24:06.999999Z");
+}
+
+#[test]
+fn google_ei_matches_only_the_ei_and_sei_parameter_names() {
+    // `sei=` carries the same encoding (Cheeky4n6Monkey 2014: sei and ei from one
+    // session share their leading bytes).
+    let (r, _) =
+        ei_reading("https://www.google.com.au/search?q=bananas&gbv=1&sei=BrU2VKfrB9Xz8gX2iILoBA");
+    assert!(r.starts_with("2014-10-09T16:17:10"), "{r}");
+    // A parameter merely ENDING in "ei" is a different parameter; a value that is
+    // not urlsafe base64, or decodes to under the 4 seconds bytes, is no reading.
+    for other in [
+        "?gei=ttqdXsP7IMKZk74Pgv-k6AY",
+        "x?q=1&rei=ttqdXsP7IMKZk74Pgv-k6AY",
+        "ei=ttqd+sP7",
+        "ei=ttqd",
+    ] {
+        assert!(
+            !interpret::interpret_string(other)
+                .iter()
+                .any(|c| c.format_id == "google_ei"),
+            "{other} is not an ei= parameter"
+        );
+    }
+}
+
 #[test]
 fn apache_clf_datetime() {
     // Apache/nginx common-log-format date, with and without the surrounding

tests/unfurl_oracle.rs

@@ -63,3 +63,24 @@ fn unfurl_agrees_on_discord_snowflake() {
         "unfurl vs timeglyph discord (ms)"
     );
 }
+
+#[test]
+fn unfurl_agrees_on_google_ei_to_the_microsecond() {
+    // unfurl joins ei's seconds and microsecond varint into one `ei Timestamp:`
+    // in Unix MICROseconds (the helper's name notwithstanding). Real URL from
+    // unfurl issue #56.
+    let url = "https://www.google.com/search?ei=ttqdXsP7IMKZk74Pgv-k6AY&q=x";
+    let Some(us) = unfurl_timestamp_ms(url) else {
+        eprintln!("unfurl unavailable — skipping (install dfir-unfurl to run)");
+        return;
+    };
+    let mine = timeglyph::interpret::interpret_string(url)
+        .into_iter()
+        .find(|c| c.format_id == "google_ei")
+        .expect("timeglyph google_ei candidate");
+    assert_eq!(
+        i128::from(us),
+        mine.instant.0 / 1_000,
+        "unfurl vs timeglyph google_ei (µs)"
+    );
+}