From e400fcaf28f860e11fc3ca358b0255a534061e3d Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:23:06 +0100 Subject: [PATCH] Port Python JSON byte transport (3f-2b-1) --- README.md | 7 +- .../src/evidence/legacy_verify.rs | 1 + .../evidence/legacy_verify/statement_bytes.rs | 89 +++++++++++++++++++ .../legacy_verify/statement_bytes_tests.rs | 66 ++++++++++++++ .../evidence/legacy_verify/statement_tests.rs | 6 ++ scripts/PROVENANCE | 4 +- 6 files changed, 170 insertions(+), 3 deletions(-) create mode 100644 crates/mergemaster-evidence/src/evidence/legacy_verify/statement_bytes.rs create mode 100644 crates/mergemaster-evidence/src/evidence/legacy_verify/statement_bytes_tests.rs diff --git a/README.md b/README.md index 76e54e0..8ee3e76 100644 --- a/README.md +++ b/README.md @@ -536,8 +536,11 @@ batch note-entry presence only; neither establishes statement or signer trust. discovery (3f-1), including SHA-1/SHA-256 suffix identity and tag-first deduplication. `statement::statement_of` validates decoded statements (3f-2a), preserving raw number lexemes and requiring zoned Python ISO calendar/week timestamps. -Remaining 3f-2b: Python-compatible JSON byte decoding (including UTF-16/32, -extensions, lone surrogates and nesting) before this guard; no decoder cutover yet. +`statement_bytes::decode_json_bytes` ports Python's BOM/NUL-based UTF-8/16/32 +transport detection and surrogatepass decoding (3f-2b-1), retaining code points +including lone surrogates rather than replacing them with invalid Rust strings. +Remaining 3f-2b-2: JSON grammar/values, extensions, surrogate object keys and +parser limits, then byte-to-statement integration; no decoder cutover yet. Then 3f-3 Git record layout/parent/subject checks and owned cache (`read_statement` and `load`, 720-765). Signature verification (768-773) follows before selection (3g) and require-all (3h); none of these candidate indexes establishes trust. diff --git a/crates/mergemaster-evidence/src/evidence/legacy_verify.rs b/crates/mergemaster-evidence/src/evidence/legacy_verify.rs index f7a464d..b455d9a 100644 --- a/crates/mergemaster-evidence/src/evidence/legacy_verify.rs +++ b/crates/mergemaster-evidence/src/evidence/legacy_verify.rs @@ -11,6 +11,7 @@ pub mod routing; pub mod selection; pub mod selection_json; pub mod statement; +pub mod statement_bytes; pub mod workflow; use super::legacy_policy::kind_rules::KindRule; use serde::Serialize; diff --git a/crates/mergemaster-evidence/src/evidence/legacy_verify/statement_bytes.rs b/crates/mergemaster-evidence/src/evidence/legacy_verify/statement_bytes.rs new file mode 100644 index 0000000..d1e76b5 --- /dev/null +++ b/crates/mergemaster-evidence/src/evidence/legacy_verify/statement_bytes.rs @@ -0,0 +1,89 @@ +//! Python json.loads byte transport, before JSON parsing or statement validation. +/// Python strings may contain lone surrogates, so retain code points, not Rust chars. +/// Matches json.detect_encoding followed by decoding with errors="surrogatepass". +pub fn decode_json_bytes(bytes: &[u8]) -> Option> { + let (data, width, little) = if bytes.starts_with(b"\0\0\xfe\xff") { + (&bytes[4..], 4, false) + } else if bytes.starts_with(b"\xff\xfe\0\0") { + (&bytes[4..], 4, true) + } else if bytes.starts_with(b"\xfe\xff") { + (&bytes[2..], 2, false) + } else if bytes.starts_with(b"\xff\xfe") { + (&bytes[2..], 2, true) + } else if bytes.starts_with(b"\xef\xbb\xbf") { + (&bytes[3..], 1, false) + } else if bytes.len() >= 4 && bytes[0] == 0 { + (bytes, if bytes[1] == 0 { 4 } else { 2 }, false) + } else if bytes.len() >= 4 && bytes[1] == 0 { + ( + bytes, + if bytes[2] == 0 && bytes[3] == 0 { 4 } else { 2 }, + true, + ) + } else if bytes.len() == 2 && (bytes[0] == 0 || bytes[1] == 0) { + (bytes, 2, bytes[1] == 0) + } else { + (bytes, 1, false) + }; + if !data.len().is_multiple_of(width) { + return None; + } + let mut points = Vec::with_capacity(data.len() / width); + let mut i = 0; + while i < data.len() { + let mut point = u32::from(data[i]); + if width == 1 { + let (length, minimum) = match data[i] { + 0..=0x7f => (1, 0), + 0xc2..=0xdf => { + point &= 0x1f; + (2, 0x80) + } + 0xe0..=0xef => { + point &= 0x0f; + (3, 0x800) + } + 0xf0..=0xf4 => { + point &= 7; + (4, 0x10000) + } + _ => return None, + }; + for &byte in data.get(i + 1..i + length)? { + if byte & 0xc0 != 0x80 { + return None; + } + point = (point << 6) | u32::from(byte & 0x3f); + } + if !(minimum..=0x10ffff).contains(&point) { + return None; + } + i += length; + } else { + point = 0; + for (j, &byte) in data[i..i + width].iter().enumerate() { + point |= u32::from(byte) << (8 * if little { j } else { width - j - 1 }); + } + i += width; + if width == 2 && (0xd800..=0xdbff).contains(&point) && i < data.len() { + let next = if little { + u16::from_le_bytes([data[i], data[i + 1]]) + } else { + u16::from_be_bytes([data[i], data[i + 1]]) + }; + if (0xdc00..=0xdfff).contains(&next) { + point = 0x10000 + ((point - 0xd800) << 10) + u32::from(next - 0xdc00); + i += 2; + } + } + if point > 0x10ffff { + return None; + } + } + points.push(point); + } + Some(points) +} +#[cfg(test)] +#[path = "statement_bytes_tests.rs"] +mod tests; diff --git a/crates/mergemaster-evidence/src/evidence/legacy_verify/statement_bytes_tests.rs b/crates/mergemaster-evidence/src/evidence/legacy_verify/statement_bytes_tests.rs new file mode 100644 index 0000000..1ddf611 --- /dev/null +++ b/crates/mergemaster-evidence/src/evidence/legacy_verify/statement_bytes_tests.rs @@ -0,0 +1,66 @@ +use super::decode_json_bytes; +#[test] +fn legacy_statement_bytes_match_python_transport() { + let root = std::env::var("MERGEMASTER_PYTHON_ORACLE").expect("pinned Python oracle required"); + let script = r#" +import copy,json,runpy,sys +oracle=runpy.run_path(sys.argv[1]+'/scripts/evidence-verify') +tree='a'*40 +valid={'_type':oracle['STATEMENT_TYPE'],'predicateType':oracle['PREDICATE_TYPE'], + 'subject':[{'name':'b'*40,'digest':{'gitTree':tree}}], + 'predicate':{'schema_version':1,'change_id':'change🌍','tier':'full','signer':None, + 'observed_at':'2026-01-01T00:00:00Z','observed':{'kind':'tests','runner':'wrapper', + 'checks':{'check':{'command':[''],'exit_code':0,'output_sha256':'c'*64}}}}} +text=json.dumps(valid,ensure_ascii=False) +cases=[] +for encoding in ['utf-8','utf-8-sig','utf-16','utf-16-le','utf-16-be','utf-32','utf-32-le','utf-32-be']: + cases.append(('valid-'+encoding,text.encode(encoding))) +for encoding,bom in [('utf-16-be',b'\xfe\xff'),('utf-32-be',b'\0\0\xfe\xff')]: + cases.append(('valid-bom-'+encoding,bom+text.encode(encoding))) +boundary=json.dumps('x'+''.join(map(chr,[0,0x7f,0x80,0x7ff,0x800,0xd7ff,0xd800,0xdbff,0xdc00,0xdfff,0xe000,0xffff,0x10000,0x10ffff])),ensure_ascii=False) +for encoding in ['utf-8','utf-16-le','utf-16-be','utf-32-le','utf-32-be']: + cases.append(('boundaries-'+encoding,boundary.encode(encoding,'surrogatepass'))) +for name,value in [('surrogate-value','\ud800'),('surrogate-key','\udfff'),('nan',float('nan')),('infinity',float('inf')),('negative-infinity',-float('inf'))]: + v=copy.deepcopy(valid) + if name=='surrogate-key': v[value]='kept' + else: v['extra']=value + for encoding in ['utf-8','utf-16-le','utf-32-be']: + cases.append((name+'-'+encoding,json.dumps(v,ensure_ascii=False).encode(encoding,'surrogatepass'))) +for name,data in { + 'empty':'','utf8-bom-only':'efbbbf','utf16-bom-only':'fffe','utf32-bom-only':'fffe0000', + 'short-be':'0030','short-le':'3000','three-byte-nul':'003030','four-byte-nul':'00003030', + 'odd-utf16':'fffe30','truncated-utf32':'fffe00003000','utf32-too-large':'fffe000000001100', + 'utf8-overlong':'c080','utf8-overlong-three':'e08080','utf8-overlong-four':'f0808080', + 'utf8-too-large':'f4908080','utf8-truncated':'f09f8c','utf8-continuation':'80', + 'utf8-high-surrogate':'eda080','utf8-low-surrogate':'edbfbf','utf8-surrogate-pair':'eda080edb080', + 'utf16-high-surrogate':'fffe00d8','utf16-low-surrogate':'fffeffdf','utf16-pair':'fffe00d800dc', + 'utf16-unpaired-then-ascii':'fffe00d86100','utf32-surrogate':'0000feff0000d800', + 'utf8-double-bom':'efbbbfefbbbf','utf16-double-bom':'fffefffe','utf32-double-bom':'fffe0000fffe0000' +}.items(): cases.append((name,bytes.fromhex(data))) +for count in [4300,4301]: cases.append(('integer-'+str(count),(text[:-1]+',"extra":'+'9'*count+'}').encode())) +for count in [100,2000]: cases.append(('depth-'+str(count),(text[:-1]+',"extra":'+'['*count+'0'+']'*count+'}').encode())) +results=[] +for name,blob in cases: + try: points=list(map(ord,blob.decode(json.detect_encoding(blob),'surrogatepass'))) + except UnicodeError: points=None + try: statement=oracle['statement_of'](blob,tree) is not None + except Exception as e: statement=type(e).__name__ + results.append(dict(name=name,bytes=list(blob),points=points,statement=statement)) +print(json.dumps(results)) +"#; + let output = std::process::Command::new("python3") + .args(["-c", script, &root]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let cases: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + for case in cases.as_array().unwrap() { + let bytes: Vec = serde_json::from_value(case["bytes"].clone()).unwrap(); + let expected: Option> = serde_json::from_value(case["points"].clone()).unwrap(); + assert_eq!(decode_json_bytes(&bytes), expected, "{}", case["name"]); + } +} diff --git a/crates/mergemaster-evidence/src/evidence/legacy_verify/statement_tests.rs b/crates/mergemaster-evidence/src/evidence/legacy_verify/statement_tests.rs index c6c5ba3..a8b93e7 100644 --- a/crates/mergemaster-evidence/src/evidence/legacy_verify/statement_tests.rs +++ b/crates/mergemaster-evidence/src/evidence/legacy_verify/statement_tests.rs @@ -98,4 +98,10 @@ print(json.dumps([o['statement_of'](s.encode(),sys.argv[2]) is not None for s in ) .unwrap(); assert_eq!(parsed["predicate"]["change_id"], "change"); + assert_eq!( + parsed["predicate"]["observed"]["checks"]["check"]["command"] + .as_array() + .unwrap()[0], + "" + ); } diff --git a/scripts/PROVENANCE b/scripts/PROVENANCE index 5005104..a964657 100644 --- a/scripts/PROVENANCE +++ b/scripts/PROVENANCE @@ -30,7 +30,9 @@ # widths of 40 or 64; legacy callers derive these from validated tree IDs. # Rust parity: statement shape, 3f-2a (637-665): decoded tests records, zoned ISO # calendar/week clocks, schema bool/float equality and integer-only exit codes. -# JSON byte decoding (635-636,666-667) remains 3f-2b, not a silently narrower decoder. +# Rust parity: 3f-2b-1 ports json.loads' byte transport at 636: BOM/NUL encoding +# detection and UTF-8/16/32 surrogatepass decoding, returning Python code points. +# JSON grammar/values and exception behavior (635-636,666-667) remain 3f-2b-2. # Rust parity: workflow-job-selection (462-520). Raw top-level/job bytes and # dependency closure match; workflow layout is not interpreted as general YAML. # Shared legacy JSON guards/encoding preserve lane behavior; native and push