Port legacy statement byte transport (3f-2b-1)

README.md

@@ -536,8 +536,11 @@ batch note-entry presence only; neither establishes statement or signer trust.
 discovery (3f-1), including SHA-1/SHA-256 suffix identity and tag-first deduplication.
 `statement::statement_of` validates decoded statements (3f-2a), preserving raw
 number lexemes and requiring zoned Python ISO calendar/week timestamps.
-Remaining 3f-2b: Python-compatible JSON byte decoding (including UTF-16/32,
-extensions, lone surrogates and nesting) before this guard; no decoder cutover yet.
+`statement_bytes::decode_json_bytes` ports Python's BOM/NUL-based UTF-8/16/32
+transport detection and surrogatepass decoding (3f-2b-1), retaining code points
+including lone surrogates rather than replacing them with invalid Rust strings.
+Remaining 3f-2b-2: JSON grammar/values, extensions, surrogate object keys and
+parser limits, then byte-to-statement integration; no decoder cutover yet.
 Then 3f-3 Git record layout/parent/subject checks and owned cache (`read_statement`
 and `load`, 720-765). Signature verification (768-773) follows before selection
 (3g) and require-all (3h); none of these candidate indexes establishes trust.

crates/mergemaster-evidence/src/evidence/legacy_verify.rs

@@ -11,6 +11,7 @@ pub mod routing;
 pub mod selection;
 pub mod selection_json;
 pub mod statement;
+pub mod statement_bytes;
 pub mod workflow;
 use super::legacy_policy::kind_rules::KindRule;
 use serde::Serialize;

crates/mergemaster-evidence/src/evidence/legacy_verify/statement_bytes.rs

@@ -0,0 +1,89 @@
+//! Python json.loads byte transport, before JSON parsing or statement validation.
+/// Python strings may contain lone surrogates, so retain code points, not Rust chars.
+/// Matches json.detect_encoding followed by decoding with errors="surrogatepass".
+pub fn decode_json_bytes(bytes: &[u8]) -> Option<Vec<u32>> {
+    let (data, width, little) = if bytes.starts_with(b"\0\0\xfe\xff") {
+        (&bytes[4..], 4, false)
+    } else if bytes.starts_with(b"\xff\xfe\0\0") {
+        (&bytes[4..], 4, true)
+    } else if bytes.starts_with(b"\xfe\xff") {
+        (&bytes[2..], 2, false)
+    } else if bytes.starts_with(b"\xff\xfe") {
+        (&bytes[2..], 2, true)
+    } else if bytes.starts_with(b"\xef\xbb\xbf") {
+        (&bytes[3..], 1, false)
+    } else if bytes.len() >= 4 && bytes[0] == 0 {
+        (bytes, if bytes[1] == 0 { 4 } else { 2 }, false)
+    } else if bytes.len() >= 4 && bytes[1] == 0 {
+        (
+            bytes,
+            if bytes[2] == 0 && bytes[3] == 0 { 4 } else { 2 },
+            true,
+        )
+    } else if bytes.len() == 2 && (bytes[0] == 0 || bytes[1] == 0) {
+        (bytes, 2, bytes[1] == 0)
+    } else {
+        (bytes, 1, false)
+    };
+    if !data.len().is_multiple_of(width) {
+        return None;
+    }
+    let mut points = Vec::with_capacity(data.len() / width);
+    let mut i = 0;
+    while i < data.len() {
+        let mut point = u32::from(data[i]);
+        if width == 1 {
+            let (length, minimum) = match data[i] {
+                0..=0x7f => (1, 0),
+                0xc2..=0xdf => {
+                    point &= 0x1f;
+                    (2, 0x80)
+                }
+                0xe0..=0xef => {
+                    point &= 0x0f;
+                    (3, 0x800)
+                }
+                0xf0..=0xf4 => {
+                    point &= 7;
+                    (4, 0x10000)
+                }
+                _ => return None,
+            };
+            for &byte in data.get(i + 1..i + length)? {
+                if byte & 0xc0 != 0x80 {
+                    return None;
+                }
+                point = (point << 6) | u32::from(byte & 0x3f);
+            }
+            if !(minimum..=0x10ffff).contains(&point) {
+                return None;
+            }
+            i += length;
+        } else {
+            point = 0;
+            for (j, &byte) in data[i..i + width].iter().enumerate() {
+                point |= u32::from(byte) << (8 * if little { j } else { width - j - 1 });
+            }
+            i += width;
+            if width == 2 && (0xd800..=0xdbff).contains(&point) && i < data.len() {
+                let next = if little {
+                    u16::from_le_bytes([data[i], data[i + 1]])
+                } else {
+                    u16::from_be_bytes([data[i], data[i + 1]])
+                };
+                if (0xdc00..=0xdfff).contains(&next) {
+                    point = 0x10000 + ((point - 0xd800) << 10) + u32::from(next - 0xdc00);
+                    i += 2;
+                }
+            }
+            if point > 0x10ffff {
+                return None;
+            }
+        }
+        points.push(point);
+    }
+    Some(points)
+}
+#[cfg(test)]
+#[path = "statement_bytes_tests.rs"]
+mod tests;

crates/mergemaster-evidence/src/evidence/legacy_verify/statement_bytes_tests.rs

@@ -0,0 +1,66 @@
+use super::decode_json_bytes;
+#[test]
+fn legacy_statement_bytes_match_python_transport() {
+    let root = std::env::var("MERGEMASTER_PYTHON_ORACLE").expect("pinned Python oracle required");
+    let script = r#"
+import copy,json,runpy,sys
+oracle=runpy.run_path(sys.argv[1]+'/scripts/evidence-verify')
+tree='a'*40
+valid={'_type':oracle['STATEMENT_TYPE'],'predicateType':oracle['PREDICATE_TYPE'],
+ 'subject':[{'name':'b'*40,'digest':{'gitTree':tree}}],
+ 'predicate':{'schema_version':1,'change_id':'change🌍','tier':'full','signer':None,
+ 'observed_at':'2026-01-01T00:00:00Z','observed':{'kind':'tests','runner':'wrapper',
+ 'checks':{'check':{'command':[''],'exit_code':0,'output_sha256':'c'*64}}}}}
+text=json.dumps(valid,ensure_ascii=False)
+cases=[]
+for encoding in ['utf-8','utf-8-sig','utf-16','utf-16-le','utf-16-be','utf-32','utf-32-le','utf-32-be']:
+ cases.append(('valid-'+encoding,text.encode(encoding)))
+for encoding,bom in [('utf-16-be',b'\xfe\xff'),('utf-32-be',b'\0\0\xfe\xff')]:
+ cases.append(('valid-bom-'+encoding,bom+text.encode(encoding)))
+boundary=json.dumps('x'+''.join(map(chr,[0,0x7f,0x80,0x7ff,0x800,0xd7ff,0xd800,0xdbff,0xdc00,0xdfff,0xe000,0xffff,0x10000,0x10ffff])),ensure_ascii=False)
+for encoding in ['utf-8','utf-16-le','utf-16-be','utf-32-le','utf-32-be']:
+ cases.append(('boundaries-'+encoding,boundary.encode(encoding,'surrogatepass')))
+for name,value in [('surrogate-value','\ud800'),('surrogate-key','\udfff'),('nan',float('nan')),('infinity',float('inf')),('negative-infinity',-float('inf'))]:
+ v=copy.deepcopy(valid)
+ if name=='surrogate-key': v[value]='kept'
+ else: v['extra']=value
+ for encoding in ['utf-8','utf-16-le','utf-32-be']:
+  cases.append((name+'-'+encoding,json.dumps(v,ensure_ascii=False).encode(encoding,'surrogatepass')))
+for name,data in {
+ 'empty':'','utf8-bom-only':'efbbbf','utf16-bom-only':'fffe','utf32-bom-only':'fffe0000',
+ 'short-be':'0030','short-le':'3000','three-byte-nul':'003030','four-byte-nul':'00003030',
+ 'odd-utf16':'fffe30','truncated-utf32':'fffe00003000','utf32-too-large':'fffe000000001100',
+ 'utf8-overlong':'c080','utf8-overlong-three':'e08080','utf8-overlong-four':'f0808080',
+ 'utf8-too-large':'f4908080','utf8-truncated':'f09f8c','utf8-continuation':'80',
+ 'utf8-high-surrogate':'eda080','utf8-low-surrogate':'edbfbf','utf8-surrogate-pair':'eda080edb080',
+ 'utf16-high-surrogate':'fffe00d8','utf16-low-surrogate':'fffeffdf','utf16-pair':'fffe00d800dc',
+ 'utf16-unpaired-then-ascii':'fffe00d86100','utf32-surrogate':'0000feff0000d800',
+ 'utf8-double-bom':'efbbbfefbbbf','utf16-double-bom':'fffefffe','utf32-double-bom':'fffe0000fffe0000'
+}.items(): cases.append((name,bytes.fromhex(data)))
+for count in [4300,4301]: cases.append(('integer-'+str(count),(text[:-1]+',"extra":'+'9'*count+'}').encode()))
+for count in [100,2000]: cases.append(('depth-'+str(count),(text[:-1]+',"extra":'+'['*count+'0'+']'*count+'}').encode()))
+results=[]
+for name,blob in cases:
+ try: points=list(map(ord,blob.decode(json.detect_encoding(blob),'surrogatepass')))
+ except UnicodeError: points=None
+ try: statement=oracle['statement_of'](blob,tree) is not None
+ except Exception as e: statement=type(e).__name__
+ results.append(dict(name=name,bytes=list(blob),points=points,statement=statement))
+print(json.dumps(results))
+"#;
+    let output = std::process::Command::new("python3")
+        .args(["-c", script, &root])
+        .output()
+        .unwrap();
+    assert!(
+        output.status.success(),
+        "{}",
+        String::from_utf8_lossy(&output.stderr)
+    );
+    let cases: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap();
+    for case in cases.as_array().unwrap() {
+        let bytes: Vec<u8> = serde_json::from_value(case["bytes"].clone()).unwrap();
+        let expected: Option<Vec<u32>> = serde_json::from_value(case["points"].clone()).unwrap();
+        assert_eq!(decode_json_bytes(&bytes), expected, "{}", case["name"]);
+    }
+}

crates/mergemaster-evidence/src/evidence/legacy_verify/statement_tests.rs

@@ -98,4 +98,10 @@ print(json.dumps([o['statement_of'](s.encode(),sys.argv[2]) is not None for s in
     )
     .unwrap();
     assert_eq!(parsed["predicate"]["change_id"], "change");
+    assert_eq!(
+        parsed["predicate"]["observed"]["checks"]["check"]["command"]
+            .as_array()
+            .unwrap()[0],
+        ""
+    );
 }

scripts/PROVENANCE

@@ -30,7 +30,9 @@
 # widths of 40 or 64; legacy callers derive these from validated tree IDs.
 # Rust parity: statement shape, 3f-2a (637-665): decoded tests records, zoned ISO
 # calendar/week clocks, schema bool/float equality and integer-only exit codes.
-# JSON byte decoding (635-636,666-667) remains 3f-2b, not a silently narrower decoder.
+# Rust parity: 3f-2b-1 ports json.loads' byte transport at 636: BOM/NUL encoding
+# detection and UTF-8/16/32 surrogatepass decoding, returning Python code points.
+# JSON grammar/values and exception behavior (635-636,666-667) remain 3f-2b-2.
 # Rust parity: workflow-job-selection (462-520). Raw top-level/job bytes and
 # dependency closure match; workflow layout is not interpreted as general YAML.
 # Shared legacy JSON guards/encoding preserve lane behavior; native and push