diff --git a/recipe/vela/native.md b/recipe/vela/native.md index 46eb71d5..b69be7ec 100644 --- a/recipe/vela/native.md +++ b/recipe/vela/native.md @@ -95,6 +95,15 @@ re-run `export_head_inputs.py` (add `--device cuda` to record the bf16-autocast numbers the member-calibrated gates expect) before the opt-in head/response/executor replays below. +For an acceptance run that preserves committed fixture history, record into a +separate output directory and use an isolated test checkout with the complete +current JSON/tensor pairs. Freeze their hashes before native comparison. Do not +combine freshly recorded hidden tensors with another run's JSON expectations; +report their differences from committed history separately. The recorder's +execution order can differ from planned row order, so attributed executor calls +match the recorded row, kind and span head, with every call consumed exactly +once. Missing or duplicate matches fail; numerical and shape gates stay fixed. + The token plans record what the pinned runtime feeds the backbone (the predict mapping, the per-row decomposition, the rendered `p_ids`, blocks and readout positions, truncation, a windowed long-span case). diff --git a/src/models/vela/README.md b/src/models/vela/README.md index aa87b9ec..447797fe 100644 --- a/src/models/vela/README.md +++ b/src/models/vela/README.md @@ -297,7 +297,7 @@ every decoder member: hidden, bilinear dim, slot count — derived from the tensors' own shapes, so one implementation serves hidden 1024/2560/4096 alike; the executor cross-checks the backbone's hidden size against them) - and the scoring + and reject non-finite head tensor values before scoring. The scoring path: CandidateHead, both SpanHeadV2 heads, `set_bias`, the per-type temperatures, set thresholds, the PII length rule and sparse gate, `span_decode_r4` (word units, unit vote, fill, @@ -306,19 +306,23 @@ every decoder member: SystemOne envelope (Noul views with the range-filtered max word probability and set fan-out, span clipping to the plan's field ranges with re-trim, `span_heads`, thresholds, confidence formulas, - usage). Replaying the recorded head-input fixtures + usage). The historical 0.8B replay of recorded head-input fixtures (`tests/vela/fixtures/head-input-*`): head-logit drift ≤ 3.8e-5, predict-answer drift ≤ 7.3e-7, and the finished responses match the frozen oracles within 1.5e-6 on four cases. The windowed long-span fixture's reference answer carries no spans, so its drift comparison covers the per-window head logits and the empty result, not the - window-aggregation arithmetic itself; a windowed fixture whose - reference answer has spans would close that gap. + window-aggregation arithmetic itself. The larger members have + separately labeled GPU-recorded fixtures: their positive long-window + expectations contain 76 spans for 4B and 373 for 9B. Their strict + head and response replays must be qualified per member and hardware + profile; the 0.8B measurements do not establish that qualification. - `native/src/executor.rs` runs the backbone on CUDA through the shared qwen3_5 crate: the family `Model` exposes only last-position hidden states, so every readout position runs as its own full - sequence — `[parts .. position]` — which causality makes identical - to the reference's in-tree readout, packed into `forward_batch`. + sequence — `[parts .. position]` — through an independent + `forward_fixed` call (fixed-M=64 GEMMs), keeping the reference's + causal readout positions without packing the backbone calls. The backbone runs from the uniform-BF16 export (the shared loader's format); the heads stay F32 on the faithful tensors. Recorded execution on sm_120 (RTX 5060, CUDA 13.1): the four small frozen @@ -342,6 +346,6 @@ widens native support. Multi-part span targets remain an explicit native The crate is a library; the HTTP worker around `Processor` + `Executor` is not part of it. Long-span GPU parity and measured latency are not recorded. Checks: `cargo test -p omni-vela-native` -(56 tests, no weights, no GPU; render, head and response parity are +(ordinary tests need no weights or GPU; render, head and response parity are opt-in with `VELA_NATIVE_BUNDLE`, executor parity additionally with `VELA_CUDA_LIB`; see [recipe/vela/native.md](../../recipe/vela/native.md)). diff --git a/src/models/vela/native/src/heads.rs b/src/models/vela/native/src/heads.rs index 20587c9b..bc20ae8a 100644 --- a/src/models/vela/native/src/heads.rs +++ b/src/models/vela/native/src/heads.rs @@ -238,14 +238,21 @@ fn tensor<'a>(file: &'a SafeTensors<'a>, name: &str) -> Result { view.dtype() == safetensors::Dtype::F32, "{name} must be F32" ); + let (chunks, remainder) = view.data().as_chunks::<4>(); + ensure!( + remainder.is_empty(), + "{name} has an invalid F32 byte length" + ); + let data: Vec = chunks + .iter() + .map(|chunk| f32::from_le_bytes(*chunk)) + .collect(); + ensure!( + data.iter().all(|value| value.is_finite()), + "{name} contains non-finite values" + ); Ok(Tensor { - data: view - .data() - .as_chunks::<4>() - .0 - .iter() - .map(|chunk| f32::from_le_bytes(*chunk)) - .collect(), + data, shape: view.shape().to_vec(), }) } diff --git a/tests/vela/native_executor.rs b/tests/vela/native_executor.rs index fb1098d2..d29e3fa6 100644 --- a/tests/vela/native_executor.rs +++ b/tests/vela/native_executor.rs @@ -110,7 +110,7 @@ fn the_gpu_backbone_replays_the_reference_readouts() { )); let _file = safetensors::SafeTensors::deserialize(&raw).expect("safetensors"); - // The recorded reference logits per call, in call order. + // The recorder's execution order can differ from planned row order. let calls: Vec<&Value> = head_fixture["head_calls"] .as_array() .unwrap() @@ -118,14 +118,16 @@ fn the_gpu_backbone_replays_the_reference_readouts() { .collect(); let mut q_call_index = 0usize; let mut span_call_index = 0usize; + let mut consumed = vec![false; calls.len()]; let mut worst_logit = 0.0f32; - for row in &planned.rows { + for (row_index, row) in planned.rows.iter().enumerate() { let readouts = executor.row_readouts(row).expect("readouts"); // One recorded q call carries the row's question blocks // stacked; compare per block, matched by option count. if !readouts.questions.is_empty() { - let call = next_call(&calls, &mut q_call_index, "q"); + let call = next_call(&calls, &mut consumed, row_index, "q", None); + q_call_index += 1; let want: Vec> = call["logits"] .as_array() .unwrap() @@ -181,7 +183,8 @@ fn the_gpu_backbone_replays_the_reference_readouts() { &executor.heads.router }; let got = head.forward(&span.words, &span.labels); - let call = next_call(&calls, &mut span_call_index, "span"); + let call = next_call(&calls, &mut consumed, row_index, "span", Some(&which)); + span_call_index += 1; let want: Vec> = call["logits"] .as_array() .unwrap() @@ -213,6 +216,10 @@ fn the_gpu_backbone_replays_the_reference_readouts() { calls.iter().filter(|c| c["kind"] == "span").count(), "{case}: span calls" ); + assert!( + consumed.iter().all(|used| *used), + "{case}: unused head call" + ); println!("{case}: worst logit drift {worst_logit:.4}"); assert!( worst_logit as f64 <= logit_gate, @@ -221,12 +228,101 @@ fn the_gpu_backbone_replays_the_reference_readouts() { } } -fn next_call<'a>(calls: &[&'a Value], index: &mut usize, kind: &str) -> &'a Value { - let call = calls - .iter() - .filter(|c| c["kind"] == kind) - .nth(*index) - .unwrap_or_else(|| panic!("recorded {kind} call")); - *index += 1; - call +fn next_call<'a>( + calls: &[&'a Value], + consumed: &mut [bool], + row: usize, + kind: &str, + head: Option<&str>, +) -> &'a Value { + assert_eq!(calls.len(), consumed.len()); + let attributed = calls.iter().any(|call| call.get("row").is_some()); + let index = if attributed { + assert!( + calls.iter().all(|call| call["row"].as_u64().is_some()), + "all recorded calls must carry valid row attribution" + ); + let matches: Vec = calls + .iter() + .enumerate() + .filter(|(index, call)| { + !consumed[*index] + && call["row"].as_u64() == Some(row as u64) + && call["kind"] == kind + && head.is_none_or(|head| call["head"] == head) + }) + .map(|(index, _)| index) + .collect(); + assert_eq!( + matches.len(), + 1, + "expected exactly one unconsumed row={row} kind={kind} head={head:?} call" + ); + matches[0] + } else { + // The original CPU-recorded 0.8B fixtures predate row metadata. + calls + .iter() + .enumerate() + .position(|(index, call)| !consumed[index] && call["kind"] == kind) + .unwrap_or_else(|| panic!("recorded {kind} call")) + }; + if let Some(head) = head { + assert_eq!(calls[index]["head"], head, "recorded span head"); + } + consumed[index] = true; + calls[index] +} + +#[test] +fn attributed_calls_match_rows_when_execution_is_reordered() { + let first = serde_json::json!({"kind": "span", "row": 1, "head": "router", "logits": [[27.0]]}); + let second = + serde_json::json!({"kind": "span", "row": 0, "head": "router", "logits": [[23.0]]}); + let calls = [&first, &second]; + let mut consumed = [false; 2]; + assert_eq!( + next_call(&calls, &mut consumed, 0, "span", Some("router"))["logits"], + second["logits"] + ); + assert_eq!( + next_call(&calls, &mut consumed, 1, "span", Some("router"))["logits"], + first["logits"] + ); + assert!(consumed.iter().all(|used| *used)); +} + +#[test] +fn legacy_unattributed_calls_keep_the_original_kind_order() { + let first = serde_json::json!({"kind": "span", "head": "router", "logits": [[1.0]]}); + let second = serde_json::json!({"kind": "span", "head": "router", "logits": [[2.0]]}); + let calls = [&first, &second]; + let mut consumed = [false; 2]; + assert_eq!( + next_call(&calls, &mut consumed, 0, "span", Some("router"))["logits"], + first["logits"] + ); + assert_eq!( + next_call(&calls, &mut consumed, 1, "span", Some("router"))["logits"], + second["logits"] + ); + assert!(consumed.iter().all(|used| *used)); +} + +#[test] +fn attributed_calls_reject_missing_duplicate_and_wrong_heads() { + let one = serde_json::json!({"kind": "span", "row": 0, "head": "router", "logits": [[1.0]]}); + let other_row = + serde_json::json!({"kind": "span", "row": 1, "head": "router", "logits": [[1.0]]}); + let other_head = + serde_json::json!({"kind": "span", "row": 0, "head": "broad", "logits": [[1.0]]}); + for calls in [vec![&other_row], vec![&one, &one], vec![&other_head]] { + let mut consumed = vec![false; calls.len()]; + assert!( + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + next_call(&calls, &mut consumed, 0, "span", Some("router")); + })) + .is_err() + ); + } } diff --git a/tests/vela/native_head_tensors.rs b/tests/vela/native_head_tensors.rs index 4fd06daa..977e417c 100644 --- a/tests/vela/native_head_tensors.rs +++ b/tests/vela/native_head_tensors.rs @@ -16,10 +16,11 @@ fn f32_tensor(name: &str, shape: Vec, values: &[f32]) -> Vec { fn tensor_rejects_a_shape_that_only_matches_when_flattened() { let raw = f32_tensor("head.weight", vec![4], &[0.0; 4]); let file = SafeTensors::deserialize(&raw).expect("safetensors"); - let error = match Linear::from_weight(tensor(&file, "head.weight").expect("tensor"), "head.weight") { - Ok(_) => panic!("wrong rank must fail"), - Err(error) => error, - }; + let error = + match Linear::from_weight(tensor(&file, "head.weight").expect("tensor"), "head.weight") { + Ok(_) => panic!("wrong rank must fail"), + Err(error) => error, + }; assert!(error.to_string().contains("head.weight must be 2-D")); }