From c89b7242860812d31d2f65299c408c865e5f7423 Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Tue, 10 Nov 2026 19:39:32 +0100 Subject: [PATCH 1/3] Port decoded legacy statement validation (3f-2a) --- README.md | 7 +- .../src/evidence/legacy_verify.rs | 1 + .../src/evidence/legacy_verify/statement.rs | 146 ++++++++++++++++++ .../evidence/legacy_verify/statement_tests.rs | 111 +++++++++++++ scripts/PROVENANCE | 3 + 5 files changed, 266 insertions(+), 2 deletions(-) create mode 100644 crates/mergemaster-evidence/src/evidence/legacy_verify/statement.rs create mode 100644 crates/mergemaster-evidence/src/evidence/legacy_verify/statement_tests.rs diff --git a/README.md b/README.md index 0084817..76e54e0 100644 --- a/README.md +++ b/README.md @@ -534,8 +534,11 @@ Legacy `discovery::input_refs` and `with_note_for` find indexed candidates and batch note-entry presence only; neither establishes statement or signer trust. `discovery::tree_refs` and `legacy_refs` complete tree/first-layout candidate discovery (3f-1), including SHA-1/SHA-256 suffix identity and tag-first deduplication. -Remaining record loading: 3f-2 statement validation (`statement_of`, 633-667), -then 3f-3 Git record layout/parent/subject checks and owned cache (`read_statement` +`statement::statement_of` validates decoded statements (3f-2a), preserving raw +number lexemes and requiring zoned Python ISO calendar/week timestamps. +Remaining 3f-2b: Python-compatible JSON byte decoding (including UTF-16/32, +extensions, lone surrogates and nesting) before this guard; no decoder cutover yet. +Then 3f-3 Git record layout/parent/subject checks and owned cache (`read_statement` and `load`, 720-765). Signature verification (768-773) follows before selection (3g) and require-all (3h); none of these candidate indexes establishes trust. Lane discovery uses the gate's sanitized Git command and rejects malformed or diff --git a/crates/mergemaster-evidence/src/evidence/legacy_verify.rs b/crates/mergemaster-evidence/src/evidence/legacy_verify.rs index 5527992..f7a464d 100644 --- a/crates/mergemaster-evidence/src/evidence/legacy_verify.rs +++ b/crates/mergemaster-evidence/src/evidence/legacy_verify.rs @@ -10,6 +10,7 @@ pub mod projection; pub mod routing; pub mod selection; pub mod selection_json; +pub mod statement; pub mod workflow; use super::legacy_policy::kind_rules::KindRule; use serde::Serialize; diff --git a/crates/mergemaster-evidence/src/evidence/legacy_verify/statement.rs b/crates/mergemaster-evidence/src/evidence/legacy_verify/statement.rs new file mode 100644 index 0000000..626eea8 --- /dev/null +++ b/crates/mergemaster-evidence/src/evidence/legacy_verify/statement.rs @@ -0,0 +1,146 @@ +//! Legacy tests statements; structural validity grants no signature trust. +pub use crate::evidence::legacy_json::Value; +use crate::evidence::oid; +use chrono::{Datelike, NaiveDate}; +fn nonblank(s: &str) -> bool { + !s.trim_matches(|c: char| c.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&c)) + .is_empty() +} +fn clock(s: &str, offset: bool) -> bool { + let (main, fraction) = s + .split_once(['.', ',']) + .map_or((s, None), |(a, b)| (a, Some(b))); + if fraction.is_some_and(|v| v.is_empty() || !v.bytes().all(|b| b.is_ascii_digit())) { + return false; + } + let extended = main.contains(':'); + if !(if extended { + [2, 5, 8].contains(&main.len()) + } else { + [2, 4, 6].contains(&main.len()) + }) { + return false; + } + let mut seconds = 0; + let mut number = 0; + let mut count = 0_usize; + for (i, b) in main.bytes().enumerate() { + if extended && i % 3 == 2 { + if b != b':' { + return false; + } + continue; + } + if !b.is_ascii_digit() { + return false; + } + number = number * 10 + u32::from(b - b'0'); + count += 1; + if count.is_multiple_of(2) { + if !offset && number >= if count == 2 { 24 } else { 60 } { + return false; + } + seconds += number * [3600, 60, 1][count / 2 - 1]; + number = 0; + } + } + !offset || seconds < 86400 +} +fn zoned_time(s: &str) -> bool { + // Python replaces every Z before fromisoformat, including a separator Z. + let s = if s.contains('Z') { + std::borrow::Cow::Owned(s.replace('Z', "+00:00")) + } else { + std::borrow::Cow::Borrowed(s) + }; + let date = [10, 8, 7].into_iter().find_map(|end| { + let head = s.get(..end)?; + let parsed = ["%Y-%m-%d", "%Y%m%d", "%G-W%V-%u", "%GW%V%u"] + .into_iter() + .find_map(|f| NaiveDate::parse_from_str(head, f).ok()) + .or_else(|| { + ["%G-W%V-%u", "%GW%V%u"].into_iter().find_map(|f| { + NaiveDate::parse_from_str( + &format!("{head}{}", if head.contains('-') { "-1" } else { "1" }), + f, + ) + .ok() + }) + })?; + (parsed.year() >= 1 && parsed.year() <= 9999).then_some(end) + }); + let Some(end) = date else { + return false; + }; + let Some(tail) = s + .get(end..) + .and_then(|v| v.chars().next().map(|c| &v[c.len_utf8()..])) + else { + return false; + }; + let Some(zone) = tail.find(['+', '-']) else { + return false; + }; + clock(&tail[..zone], false) && clock(&tail[zone + 1..], true) +} +/// Validate a decoded legacy tests statement, retaining unknown fields and number lexemes. +/// JSON byte decoding belongs to the record loader, not this structural guard. +pub fn statement_of(v: Value, tree: &str) -> Option { + let p = &v["predicate"]; + let subjects = v["subject"].as_array()?; + let observed = &p["observed"]; + let schema_one = matches!(&p["schema_version"], Value::Bool(true)) + || p["schema_version"] + .as_number() + .is_some_and(|n| n.parse::().ok() == Some(1.0)); + if v["_type"] != "https://in-toto.io/Statement/v1" + || !matches!( + v["predicateType"].as_str(), + Some( + "https://compounding.tech/fractal/evidence/gate/v1" + | "https://smalltalk.dev/evidence/gate/v1" + ) + ) + || subjects.len() != 1 + || oid(subjects[0]["name"].as_str()?).is_err() + || subjects[0]["digest"].as_object()?.len() != 1 + || subjects[0]["digest"]["gitTree"].as_str() != Some(tree) + || !schema_one + || !nonblank(p["change_id"].as_str()?) + || !matches!(p["tier"].as_str(), Some("full" | "degraded" | "test")) + || !matches!(p.get("signer")?, Value::Null | Value::String(_)) + || observed["kind"] != "tests" + || !observed["runner"].is_string() + || !zoned_time(p["observed_at"].as_str()?) + { + return None; + } + if let Some(inputs) = observed.get("inputs") { + for (key, value) in inputs.as_object()? { + if !nonblank(key) || !nonblank(value.as_str()?) { + return None; + } + } + } + for (name, check) in observed["checks"].as_object()? { + let command = check["command"].as_array()?; + let code = check["exit_code"].as_number()?; + let hash = check["output_sha256"].as_str()?; + if !nonblank(name) + || command.is_empty() + || !command.iter().all(Value::is_string) + || code.contains(['.', 'e', 'E']) + || code.trim_start_matches('-').len() > 4300 + || hash.len() != 64 + || !hash + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + { + return None; + } + } + Some(v) +} +#[cfg(test)] +#[path = "statement_tests.rs"] +mod tests; diff --git a/crates/mergemaster-evidence/src/evidence/legacy_verify/statement_tests.rs b/crates/mergemaster-evidence/src/evidence/legacy_verify/statement_tests.rs new file mode 100644 index 0000000..72628ec --- /dev/null +++ b/crates/mergemaster-evidence/src/evidence/legacy_verify/statement_tests.rs @@ -0,0 +1,111 @@ +use super::*; +use serde_json::{Value as Json, json}; +#[test] +fn legacy_statement_shapes_match_oracle() { + let tree = "a".repeat(40); + let valid = json!({"_type":"https://in-toto.io/Statement/v1","predicateType":"https://compounding.tech/fractal/evidence/gate/v1", + "subject":[{"name":"b".repeat(40),"digest":{"gitTree":tree}}],"predicate":{"schema_version":1,"change_id":"change","tier":"full","signer":null, + "observed_at":"2026-01-01T12:30:00Z","observed":{"kind":"tests","runner":"wrapper","checks":{"check":{"command":[""],"exit_code":0,"output_sha256":"c".repeat(64)}}}}}); + let mut cases = vec![valid.to_string(), "null".into(), "{}".into(), "[".into()]; + let mutations: Json = serde_json::from_str(r#"[ + ["/_type",["wrong",null]], ["/predicateType",["https://smalltalk.dev/evidence/gate/v1","wrong"]], + ["/subject",[[],null]], ["/subject/0/name",["bad",null]], + ["/subject/0/digest",[{"gitTree":"wrong"},{"gitTree":"a","other":"x"}]], + ["/predicate/schema_version",[true,1.0,false,2]], ["/predicate/change_id",[" \u001c","\u0085"," x ",null]], + ["/predicate/tier",["degraded","test","other"]], ["/predicate/signer",["",1]], + ["/predicate/observed/kind",["other",null]], ["/predicate/observed/runner",["",null]], + ["/predicate/observed/inputs",[{}, {"a":" b "},{" ":"v"},{"k":false},null]], + ["/predicate/observed/checks",[{}, {" ":{}},null]], + ["/predicate/observed/checks/check/command",[[],[1],["","x"]]], + ["/predicate/observed/checks/check/exit_code",[true,1.0,-5]], + ["/predicate/observed/checks/check/output_sha256",["bad",null]] + ]"#).unwrap(); + for mutation in mutations.as_array().unwrap() { + for value in mutation[1].as_array().unwrap() { + let mut v = valid.clone(); + *v.pointer_mut(mutation[0].as_str().unwrap()).unwrap() = value.clone(); + cases.push(v.to_string()); + } + } + for time in [ + "2026-01-01", + "2026-01-01T12:30:00", + "20260101T1230Z", + "2026-W01-1T12Z", + "2026W011T12Z", + "2026-W01T12Z", + "2026W01T12Z", + "2026-01-01🐍12:30:00,123456789+013045.25", + "2026-01-01Z12Z", + "2026-01-01T12.5-00", + "2026-01-01T24Z", + "2026-02-30T12Z", + "0000-01-01T12Z", + "2026-01-01T12+24:00", + "2026-01-01T12+00:99", + "2026-01-01T12+23:59:59.99", + "2026-01-01T12+", + "2026-01-01T1Z", + ] { + let mut v = valid.clone(); + v["predicate"]["observed_at"] = json!(time); + cases.push(v.to_string()); + } + for field in [ + "schema_version", + "change_id", + "tier", + "signer", + "observed_at", + "observed", + ] { + let mut v = valid.clone(); + v["predicate"].as_object_mut().unwrap().remove(field); + cases.push(v.to_string()); + } + for n in [ + "-0".to_owned(), + "9".repeat(100), + "9".repeat(4300), + "9".repeat(4301), + ] { + cases.push( + valid + .to_string() + .replace("\"exit_code\":0", &format!("\"exit_code\":{n}")), + ); + } + let root = std::env::var("MERGEMASTER_PYTHON_ORACLE").expect("pinned Python oracle required"); + let output=std::process::Command::new("python3").args(["-c",r#" +import json,runpy,sys +o=runpy.run_path(sys.argv[1]+'/scripts/evidence-verify') +print(json.dumps([o['statement_of'](s.encode(),sys.argv[2]) is not None for s in json.loads(sys.argv[3])])) +"#,&root,&tree,&json!(cases).to_string()]).output().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let expected: Vec = serde_json::from_slice(&output.stdout).unwrap(); + for (case, expected) in cases.iter().zip(expected) { + assert_eq!( + crate::evidence::legacy_json::parse(case.as_bytes()) + .ok() + .and_then(|v| statement_of(v, &tree)) + .is_some(), + expected, + "{case}" + ); + } + let parsed = statement_of( + crate::evidence::legacy_json::parse(valid.to_string().as_bytes()).unwrap(), + &tree, + ) + .unwrap(); + assert_eq!( + parsed["predicate"]["observed"]["checks"]["check"]["command"] + .as_array() + .unwrap()[0], + "" + ); +} diff --git a/scripts/PROVENANCE b/scripts/PROVENANCE index aa042a4..c05ba9e 100644 --- a/scripts/PROVENANCE +++ b/scripts/PROVENANCE @@ -17,6 +17,9 @@ # width filtering match in SHA-1 and SHA-256 repositories, including empty/GC states. # Hardening divergence: tree-ref discovery requires full OIDs and first-layout # widths of 40 or 64; legacy callers derive these from validated tree IDs. +# Rust parity: statement shape, 3f-2a (637-665): decoded tests records, zoned ISO +# calendar/week clocks, schema bool/float equality and integer-only exit codes. +# JSON byte decoding (635-636,666-667) remains 3f-2b, not a silently narrower decoder. # Rust parity: workflow-job-selection (462-520). Raw top-level/job bytes and # dependency closure match; workflow layout is not interpreted as general YAML. # Shared legacy JSON guards/encoding preserve lane behavior; native and push From 7d98ae474c7a55994718901676345287f066b6d6 Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:15:24 +0100 Subject: [PATCH 2/3] Require exact ISO date shapes and fix statement fixture inputs --- .../src/evidence/legacy_verify/statement.rs | 12 +++++++ .../evidence/legacy_verify/statement_tests.rs | 34 +++++++------------ 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/crates/mergemaster-evidence/src/evidence/legacy_verify/statement.rs b/crates/mergemaster-evidence/src/evidence/legacy_verify/statement.rs index 626eea8..2101966 100644 --- a/crates/mergemaster-evidence/src/evidence/legacy_verify/statement.rs +++ b/crates/mergemaster-evidence/src/evidence/legacy_verify/statement.rs @@ -55,6 +55,18 @@ fn zoned_time(s: &str) -> bool { }; let date = [10, 8, 7].into_iter().find_map(|end| { let head = s.get(..end)?; + if !"dddd-dd-dd|dddddddd|dddd-Wdd-d|ddddWddd|dddd-Wdd|ddddWdd" + .split('|') + .any(|shape| { + shape.len() == head.len() + && shape + .bytes() + .zip(head.bytes()) + .all(|(p, b)| (p == b'd' && b.is_ascii_digit()) || (p != b'd' && p == b)) + }) + { + return None; + } let parsed = ["%Y-%m-%d", "%Y%m%d", "%G-W%V-%u", "%GW%V%u"] .into_iter() .find_map(|f| NaiveDate::parse_from_str(head, f).ok()) diff --git a/crates/mergemaster-evidence/src/evidence/legacy_verify/statement_tests.rs b/crates/mergemaster-evidence/src/evidence/legacy_verify/statement_tests.rs index 72628ec..88b3c23 100644 --- a/crates/mergemaster-evidence/src/evidence/legacy_verify/statement_tests.rs +++ b/crates/mergemaster-evidence/src/evidence/legacy_verify/statement_tests.rs @@ -5,8 +5,14 @@ fn legacy_statement_shapes_match_oracle() { let tree = "a".repeat(40); let valid = json!({"_type":"https://in-toto.io/Statement/v1","predicateType":"https://compounding.tech/fractal/evidence/gate/v1", "subject":[{"name":"b".repeat(40),"digest":{"gitTree":tree}}],"predicate":{"schema_version":1,"change_id":"change","tier":"full","signer":null, - "observed_at":"2026-01-01T12:30:00Z","observed":{"kind":"tests","runner":"wrapper","checks":{"check":{"command":[""],"exit_code":0,"output_sha256":"c".repeat(64)}}}}}); + "observed_at":"2026-01-01T12:30:00Z","observed":{"kind":"tests","runner":"wrapper","inputs":{},"checks":{"check":{"command":[""],"exit_code":0,"output_sha256":"c".repeat(64)}}}}}); let mut cases = vec![valid.to_string(), "null".into(), "{}".into(), "[".into()]; + let mut absent_inputs = valid.clone(); + absent_inputs["predicate"]["observed"] + .as_object_mut() + .unwrap() + .remove("inputs"); + cases.push(absent_inputs.to_string()); let mutations: Json = serde_json::from_str(r#"[ ["/_type",["wrong",null]], ["/predicateType",["https://smalltalk.dev/evidence/gate/v1","wrong"]], ["/subject",[[],null]], ["/subject/0/name",["bad",null]], @@ -27,26 +33,12 @@ fn legacy_statement_shapes_match_oracle() { cases.push(v.to_string()); } } - for time in [ - "2026-01-01", - "2026-01-01T12:30:00", - "20260101T1230Z", - "2026-W01-1T12Z", - "2026W011T12Z", - "2026-W01T12Z", - "2026W01T12Z", - "2026-01-01🐍12:30:00,123456789+013045.25", - "2026-01-01Z12Z", - "2026-01-01T12.5-00", - "2026-01-01T24Z", - "2026-02-30T12Z", - "0000-01-01T12Z", - "2026-01-01T12+24:00", - "2026-01-01T12+00:99", - "2026-01-01T12+23:59:59.99", - "2026-01-01T12+", - "2026-01-01T1Z", - ] { + for time in concat!( + "2026-01-01|2026-01-01T12:30:00|20260101T1230Z|2026-W01-1T12Z|2026W011T12Z|2026-W01T12Z|2026W01T12Z|", + "2026-01-01🐍12:30:00,123456789+013045.25|2026-01-01Z12Z|2026-01-01T12.5-00|2026-01-01T24Z|2026-01-01T24:00Z|", + "2026-1-1T12:00Z|2026-W1T12Z|26-01-01T12Z|2026- 1-01T12Z|2026-02-30T12Z|0000-01-01T12Z|", + "2026-01-01T12+24:00|2026-01-01T12+00:99|2026-01-01T12+23:59:59.99|2026-01-01T12+|2026-01-01T1Z" + ).split('|') { let mut v = valid.clone(); v["predicate"]["observed_at"] = json!(time); cases.push(v.to_string()); From 57ba5a336de55e948abefb4e79d1001bb8b0b3e2 Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:26:06 +0100 Subject: [PATCH 3/3] Match pinned Python fraction and midnight clock rules --- .../src/evidence/legacy_verify/statement.rs | 21 +++++++++++++------ .../evidence/legacy_verify/statement_tests.rs | 12 +++++------ 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/crates/mergemaster-evidence/src/evidence/legacy_verify/statement.rs b/crates/mergemaster-evidence/src/evidence/legacy_verify/statement.rs index 2101966..0846d5c 100644 --- a/crates/mergemaster-evidence/src/evidence/legacy_verify/statement.rs +++ b/crates/mergemaster-evidence/src/evidence/legacy_verify/statement.rs @@ -10,7 +10,11 @@ fn clock(s: &str, offset: bool) -> bool { let (main, fraction) = s .split_once(['.', ',']) .map_or((s, None), |(a, b)| (a, Some(b))); - if fraction.is_some_and(|v| v.is_empty() || !v.bytes().all(|b| b.is_ascii_digit())) { + if fraction.is_some_and(|v| { + v.is_empty() + || !v.bytes().all(|b| b.is_ascii_digit()) + || main.len() != if main.contains(':') { 8 } else { 6 } + }) { return false; } let extended = main.contains(':'); @@ -37,14 +41,17 @@ fn clock(s: &str, offset: bool) -> bool { number = number * 10 + u32::from(b - b'0'); count += 1; if count.is_multiple_of(2) { - if !offset && number >= if count == 2 { 24 } else { 60 } { + if !offset && number >= if count == 2 { 25 } else { 60 } { return false; } seconds += number * [3600, 60, 1][count / 2 - 1]; number = 0; } } - !offset || seconds < 86400 + seconds < 86400 + || (!offset + && seconds == 86400 + && fraction.unwrap_or("").bytes().take(6).all(|b| b == b'0')) } fn zoned_time(s: &str) -> bool { // Python replaces every Z before fromisoformat, including a separator Z. @@ -79,9 +86,9 @@ fn zoned_time(s: &str) -> bool { .ok() }) })?; - (parsed.year() >= 1 && parsed.year() <= 9999).then_some(end) + (parsed.year() >= 1 && parsed.year() <= 9999).then_some((end, parsed)) }); - let Some(end) = date else { + let Some((end, date)) = date else { return false; }; let Some(tail) = s @@ -93,7 +100,9 @@ fn zoned_time(s: &str) -> bool { let Some(zone) = tail.find(['+', '-']) else { return false; }; - clock(&tail[..zone], false) && clock(&tail[zone + 1..], true) + clock(&tail[..zone], false) + && clock(&tail[zone + 1..], true) + && (!tail.starts_with("24") || date.succ_opt().is_some_and(|d| d.year() <= 9999)) } /// Validate a decoded legacy tests statement, retaining unknown fields and number lexemes. /// JSON byte decoding belongs to the record loader, not this structural guard. diff --git a/crates/mergemaster-evidence/src/evidence/legacy_verify/statement_tests.rs b/crates/mergemaster-evidence/src/evidence/legacy_verify/statement_tests.rs index 88b3c23..c6c5ba3 100644 --- a/crates/mergemaster-evidence/src/evidence/legacy_verify/statement_tests.rs +++ b/crates/mergemaster-evidence/src/evidence/legacy_verify/statement_tests.rs @@ -37,7 +37,10 @@ fn legacy_statement_shapes_match_oracle() { "2026-01-01|2026-01-01T12:30:00|20260101T1230Z|2026-W01-1T12Z|2026W011T12Z|2026-W01T12Z|2026W01T12Z|", "2026-01-01🐍12:30:00,123456789+013045.25|2026-01-01Z12Z|2026-01-01T12.5-00|2026-01-01T24Z|2026-01-01T24:00Z|", "2026-1-1T12:00Z|2026-W1T12Z|26-01-01T12Z|2026- 1-01T12Z|2026-02-30T12Z|0000-01-01T12Z|", - "2026-01-01T12+24:00|2026-01-01T12+00:99|2026-01-01T12+23:59:59.99|2026-01-01T12+|2026-01-01T1Z" + "2026-01-01T12+24:00|2026-01-01T12+00:99|2026-01-01T12+23:59:59.99|2026-01-01T12+|2026-01-01T1Z|", + "2026-01-01T1230.5Z|2026-01-01T12:30.5Z|2026-01-01T12+01.5|2026-01-01T12+0130.5|2026-01-01T12+01:30.5|", + "2026-01-01T12+013045.5|2026-01-01T12+01:30:45.5|2026-01-01T12+00:00:99|2026-01-01T12+23:59:60|", + "2026-01-01T24:00:00.0000001Z|2026-01-01T24:00:00.1Z|2026-01-01T24:00:01Z|2026-01-01T24:01Z|9999-12-31T24Z" ).split('|') { let mut v = valid.clone(); v["predicate"]["observed_at"] = json!(time); @@ -94,10 +97,5 @@ print(json.dumps([o['statement_of'](s.encode(),sys.argv[2]) is not None for s in &tree, ) .unwrap(); - assert_eq!( - parsed["predicate"]["observed"]["checks"]["check"]["command"] - .as_array() - .unwrap()[0], - "" - ); + assert_eq!(parsed["predicate"]["change_id"], "change"); }