From 6410e7634fd5e077000810ae07dc2f251b4f044d Mon Sep 17 00:00:00 2001 From: psv666 <2693925048@qq.com> Date: Fri, 20 Nov 2026 12:12:04 +0800 Subject: [PATCH 1/4] omnijev: add opt-in BF16 reference vision arithmetic Align patch, LayerNorm, rotary, head-64 attention and vision GEMM boundaries with the pinned BF16 reference. Keep legacy/JEMM behavior and optional ABI compatibility. The original 280-answer whole-model precision gate still fails; this is the bounded vision stage for #114. --- recipe/omnijev/native.md | 45 +++++ src/backends/cuda/qwen3_5/README.md | 21 +++ src/backends/cuda/qwen3_5/ops.h | 10 ++ src/backends/cuda/qwen3_5/reference_patch.cu | 14 +- src/backends/cuda/qwen3_5/reference_vision.cu | 13 ++ .../qwen3_5/reference_vision_attention.cu | 20 +++ src/models/omnijev/README.md | 15 ++ src/models/omnijev/native/src/executor.rs | 36 +++- src/models/qwen3_5/native/src/cuda.rs | 13 +- .../qwen3_5/native/src/cuda/reference.rs | 54 ++++++ src/models/qwen3_5/native/src/vision/model.rs | 156 ++++++++++++++++-- tests/omnijev/window_graph.rs | 15 ++ tests/qwen3_5/README.md | 48 ++++++ tests/qwen3_5/test_vision_omnijev.py | 135 +++++++++++++++ tests/qwen3_5/vision_batch.rs | 20 +++ tests/qwen3_5/vision_omnijev.rs | 136 +++++++++++++++ tests/qwen3_5/vision_omnijev_reference.py | 108 ++++++++++++ 17 files changed, 841 insertions(+), 18 deletions(-) create mode 100644 tests/qwen3_5/test_vision_omnijev.py create mode 100644 tests/qwen3_5/vision_omnijev.rs create mode 100644 tests/qwen3_5/vision_omnijev_reference.py diff --git a/recipe/omnijev/native.md b/recipe/omnijev/native.md index f8716919..02d148b8 100644 --- a/recipe/omnijev/native.md +++ b/recipe/omnijev/native.md @@ -19,6 +19,50 @@ cargo build --release --locked -p omni-omnijev-native OMNIJEV_MODEL=$PWD/omnijev-export target/release/omni-omnijev-native ``` +## Explicit BF16 reference vision + +`OMNIJEV_VISION_NUMERICS=reference` selects the unadapted OmniJev-4B vision +arithmetic. The default, `legacy`, retains the existing worker output. Unknown +values fail startup. The reference selection needs the optional cuDNN 9 build +and additive OmniJev CUDA symbols; requesting it with an old library fails with +a rebuild instruction. Build and launch it explicitly: + +```sh +CUDNN_INCLUDE_DIR=/absolute/cudnn/include CUDNN_LIB_DIR=/absolute/cudnn/lib \ + src/backends/cuda/qwen3_5/build.sh target/release 89 +OMNIJEV_VISION_NUMERICS=reference OMNIJEV_MODEL=$PWD/omnijev-export \ + target/release/omni-omnijev-native +``` + +This path uses a cuDNN BF16 patch convolution, vectorized Welford LayerNorm, +the existing framework GEMM policy with Torch's 1 MiB heuristic budget +(including its reduction schemes), FP32 +rotary angles/trigonometry, and head-64 reference attention. It retains BF16 +operator outputs and the separate patch-bias addition. It selects vision only: +the language weights still contain the merged LoRA, and language/readout and +calibration arithmetic are unchanged. The official unmerged BF16 model and the +original probability/decision gates remain the acceptance reference. Vision +alignment alone does not establish whole-model numerical acceptance. + +The profile uses the existing four-grid FIFO, rejects vision adapters and +explicit bounded-cache policies, and must be selected before the first forward. +`CUA_S1_VISION_GRAPH=1` can capture the vision path after warmup; descriptors, +GEMM plans and workspace are prepared before capture. Validation is specific to +the tested GPU/library stack. See the [stage and operator regressions](../../tests/qwen3_5/README.md#omnijev-4b-reference-vision). + +On one L20 (sm_89), Rust 1.100/CUDA 12.8 with cuDNN 9.24 and the documented +reference environment, the original `video.png/noul_region` case matches all +40 vision boundaries exactly, including the 24 blocks and merger. The complete +original 28-request/280-answer comparison still fails the unchanged acceptance +gate: 177 answers are within 0.01, maximum returned-field drift is 0.1758, +and three protected decisions differ (`board.jpg/q8`, `arcade.jpg/long_start`, +`web.jpg/one_token`). All 140 PNG decisions agree, but 20 PNG answers still +exceed 0.01. Unrounded values retain the same three protected failures. The +JPEG failure set changes from the legacy path; do not infer a uniform +whole-model improvement. JPEG preparation and language/LM/unmerged-LoRA +differences still need separate controlled validation. This profile stays +opt-in and does not complete #114. No performance result is claimed for it. + `CUA_S1_GRAPH=1` also enables language window Graphs in this worker, through the shared `Model::readout_window` implementation. It defaults off. Request and question prefixes and option continuations can capture and replay; current token, @@ -108,6 +152,7 @@ python3 recipe/omnijev/validate.py compare reference.json native.json options that differ in one token or share a long start, a Choice where no option fits, a 32-option Choice, a 10-level Score, and plain and region Noul. +The historical comparison and latency figures below use the legacy vision path. On one RTX 6000 Ada (sm_89, CUDA 13.2), with the reference's seven v1.1 demo stills and both question sets (140 answers), the input-token counts equal the reference's. The largest probability difference per answer, against the reference in float32 (`--fp32`) diff --git a/src/backends/cuda/qwen3_5/README.md b/src/backends/cuda/qwen3_5/README.md index 79704e27..743a2ff1 100644 --- a/src/backends/cuda/qwen3_5/README.md +++ b/src/backends/cuda/qwen3_5/README.md @@ -225,3 +225,24 @@ run the matched projection regression in `tests/qwen3_5/kernels.rs` and the [Cua screenshot qualification](../../../../recipe/cua_s1/native_screenshot.md#full-prompt-matched-algorithms) for each supported GPU/library/model combination. Other numerical profiles, vision GEMMs and the existing default/fixed policies stay on their own handles. + +## Optional OmniJev-4B vision arithmetic + +The additive ABI 9 `cs1_omnijev_vision_*` group is resolved only when explicitly +requested by `VisionModel::enable_omnijev_numerics`. Older libraries remain +loadable by existing callers. The group is separate from JEMM's guarded 27B +entry points: it accepts only unadapted 1024-wide/head-64 single-image vision, +and reuses the existing framework GEMM handle with the pinned Torch vision +1 MiB heuristic budget, without changing generic policies. +`cs1_reference_patch_create_v2(rows, hidden, stream)` adds a width-aware cuDNN +plan constructor; the original constructor still fixes hidden size to 1152. +Execute/destroy and synchronization ownership are shared. Build with cuDNN 9 +and rebuild the library before selecting the profile. + +The 4B attention workspace query returns FP32 elements for the exact bounded +row count (up to 4608), without the head-72 padding region. Q/K are compact; +V stays interleaved in QKV. Rust owns descriptor/workspace/Graph lifetimes and +rejects a bounded cache because opaque cuDNN allocations are not counted by +that cache. The default shared vision path and JEMM numerics stay unchanged. +See the [OmniJev recipe](../../../../recipe/omnijev/native.md#explicit-bf16-reference-vision) +and [regressions](../../../../tests/qwen3_5/README.md#omnijev-4b-reference-vision). diff --git a/src/backends/cuda/qwen3_5/ops.h b/src/backends/cuda/qwen3_5/ops.h index fd47a488..83d068d7 100644 --- a/src/backends/cuda/qwen3_5/ops.h +++ b/src/backends/cuda/qwen3_5/ops.h @@ -264,6 +264,16 @@ void* cs1_reference_patch_create(int rows,void* stream); int cs1_reference_patch(void* plan,const void* x,const void* weight,void* out); void cs1_reference_patch_destroy(void* plan); +// Optional OmniJev-4B vision arithmetic (additive ABI 9). Select explicitly; +// legacy/default and JEMM symbols keep their existing layouts and policies. +// Patch plans use the existing execute/destroy calls; width is 1024 or 1152. +void* cs1_reference_patch_create_v2(int rows,int hidden,void* stream); +int cs1_omnijev_vision_norm(const void* x,const void* w,const void* b,void* y,int rows,int d,void* stream); +// co contains FP32 angles, si is unused, as in the 27B reference rotary API. +int cs1_omnijev_vision_rope(const void* qkv,const float* co,const float* si,void* q,void* k,int n,int hidden,int head_dim,void* stream); +size_t cs1_omnijev_vision_attention_workspace_floats(int rows,int heads); +int cs1_omnijev_vision_attention(const void* q,const void* k,const void* v,void* out,int rows,int heads,int head_dim,void* workspace,void* stream); + #ifdef __cplusplus } #endif diff --git a/src/backends/cuda/qwen3_5/reference_patch.cu b/src/backends/cuda/qwen3_5/reference_patch.cu index 62c071cc..53e9dffd 100644 --- a/src/backends/cuda/qwen3_5/reference_patch.cu +++ b/src/backends/cuda/qwen3_5/reference_patch.cu @@ -30,9 +30,9 @@ extern "C" int cs1_reference_available() { return 0; #endif } -extern "C" void* cs1_reference_patch_create(int rows,void* stream) { +extern "C" void* cs1_reference_patch_create_v2(int rows,int hidden,void* stream) { #ifdef CS1_REFERENCE_CUDNN - if(rows<=0 || rows>65536) return nullptr; + if(rows<=0 || rows>65536 || (hidden!=1024 && hidden!=1152)) return nullptr; PatchPlan* p=new(std::nothrow) PatchPlan; if(!p) return nullptr; bool ready=false; @@ -45,8 +45,8 @@ extern "C" void* cs1_reference_patch_create(int rows,void* stream) { CHECK_CUDNN(cudnnCreateFilterDescriptor(&p->filter)); CHECK_CUDNN(cudnnCreateConvolutionDescriptor(&p->conv)); int xs[]={rows,3,2,16,16},xt[]={1536,512,256,16,1}; - int ys[]={rows,1152,1,1,1},yt[]={1152,1,1,1,1}; - int ws[]={1152,3,2,16,16},pad[]={0,0,0},step[]={2,16,16},dilation[]={1,1,1}; + int ys[]={rows,hidden,1,1,1},yt[]={hidden,1,1,1,1}; + int ws[]={hidden,3,2,16,16},pad[]={0,0,0},step[]={2,16,16},dilation[]={1,1,1}; CHECK_CUDNN(cudnnSetTensorNdDescriptor(p->input,CUDNN_DATA_BFLOAT16,5,xs,xt)); CHECK_CUDNN(cudnnSetTensorNdDescriptor(p->output,CUDNN_DATA_BFLOAT16,5,ys,yt)); CHECK_CUDNN(cudnnSetFilterNdDescriptor(p->filter,CUDNN_DATA_BFLOAT16,CUDNN_TENSOR_NCHW,5,ws)); @@ -60,9 +60,13 @@ extern "C" void* cs1_reference_patch_create(int rows,void* stream) { if(!ready) {delete p;return nullptr;} return p; #else - (void)rows;(void)stream;return nullptr; + (void)rows;(void)hidden;(void)stream;return nullptr; #endif } +// Preserve the original 27B descriptor layout and symbol. +extern "C" void* cs1_reference_patch_create(int rows,void* stream) { + return cs1_reference_patch_create_v2(rows,1152,stream); +} extern "C" int cs1_reference_patch(void* plan,const void* x,const void* w,void* y) { #ifdef CS1_REFERENCE_CUDNN auto* p=static_cast(plan); diff --git a/src/backends/cuda/qwen3_5/reference_vision.cu b/src/backends/cuda/qwen3_5/reference_vision.cu index 17dc582d..74ea9229 100644 --- a/src/backends/cuda/qwen3_5/reference_vision.cu +++ b/src/backends/cuda/qwen3_5/reference_vision.cu @@ -98,3 +98,16 @@ extern "C" int cs1_reference_vision_attention(const void* q,const void* k,const reference_vision::unpack_attention_kernel<<<((size_t)n*heads*dh+255)/256,256,0,(cudaStream_t)stream>>>(po,(bf16*)out,(size_t)n*heads*dh,dh); return cudaGetLastError(); } + +// Additive, explicit OmniJev-4B operators. The 27B guards and callers stay unchanged. +extern "C" int cs1_omnijev_vision_norm(const void* x,const void* w,const void* b,void* y,int rows,int d,void* stream) { + if(rows<=0 || d!=1024) return cudaErrorInvalidValue; + reference_vision::reference_norm_kernel<<>>((const bf16*)x,(const bf16*)w,(const bf16*)b,(bf16*)y,d); + return cudaGetLastError(); +} +extern "C" int cs1_omnijev_vision_rope(const void* qkv,const float* co,const float* si,void* q,void* k,int n,int hidden,int dh,void* stream) { + if(n<=0 || hidden!=1024 || dh!=64) return cudaErrorInvalidValue; + const size_t count=(size_t)n*hidden; + reference_vision::rope_v2_kernel<<<(count+255)/256,256,0,(cudaStream_t)stream>>>((const bf16*)qkv,co,si,(bf16*)q,(bf16*)k,count,hidden,dh); + return cudaGetLastError(); +} diff --git a/src/backends/cuda/qwen3_5/reference_vision_attention.cu b/src/backends/cuda/qwen3_5/reference_vision_attention.cu index 4c8202f5..49bf0b6c 100644 --- a/src/backends/cuda/qwen3_5/reference_vision_attention.cu +++ b/src/backends/cuda/qwen3_5/reference_vision_attention.cu @@ -194,3 +194,23 @@ extern "C" int cs1_reference_vision_attention_padded(const void* q,const void* k } extern "C" size_t cs1_reference_vision_attention_workspace_floats(int capacity,int heads) { return cs1::reference::workspace_floats(capacity,heads,80,128); } + +// The pinned 4B SDPA uses noncausal head-64 FlashAttention with 128-key tiles, +// reverse key traversal and unscaled maxima. Keep Q/K compact and V in QKV. +extern "C" size_t cs1_omnijev_vision_attention_workspace_floats(int rows,int heads) { + if(rows<=0 || rows>4608 || heads!=16) return 0; + return (size_t)rows*heads*65; +} +extern "C" int cs1_omnijev_vision_attention(const void* q,const void* k,const void* v,void* out,int rows,int heads,int dh,void* workspace,void* stream) { + using namespace cs1;namespace f=vision_reference; + if(rows<=0 || rows>4608 || heads!=16 || dh!=64 || !workspace) return cudaErrorInvalidValue; + float* memory=(float*)workspace;float* lse=memory+(size_t)rows*heads*64; + cudaFuncSetAttribute(f::flash_kernel<64,128>,cudaFuncAttributeMaxDynamicSharedMemorySize,f::SMEM_BYTES<64,128>); + f::flash_kernel<64,128><<,(cudaStream_t)stream>>>((const bf16*)q,(const bf16*)k,(const bf16*)v,heads*64*3,memory,lse,rows,heads,heads,0.125f,1); + auto status=cudaGetLastError(); + if(status==cudaSuccess) { + f::combine<<<((size_t)rows*heads*64+127)/128,128,0,(cudaStream_t)stream>>>(memory,lse,(bf16*)out,rows,heads,64,1); + status=cudaGetLastError(); + } + return status; +} diff --git a/src/models/omnijev/README.md b/src/models/omnijev/README.md index a61a5e6d..3611a99c 100644 --- a/src/models/omnijev/README.md +++ b/src/models/omnijev/README.md @@ -108,6 +108,21 @@ plus row up to rounding, execution may choose its own split points. ## Execution +`OMNIJEV_VISION_NUMERICS` explicitly selects `legacy` (the unchanged default) +or `reference`. The latter selects `VisionModel::enable_omnijev_numerics` for +the unadapted 1024-wide, head-64 vision tower. It requires a cuDNN 9 backend +build and the optional OmniJev symbols. Library callers can instead use +`Executor::load_with_vision_numerics` with `VisionNumerics::Reference`. +The [native recipe](../../../recipe/omnijev/native.md#explicit-bf16-reference-vision) +describes the build and scope. This is a vision arithmetic selection: it does +not replace the official unmerged BF16 reference, modify exported tensors, or +change language/LM/LoRA execution or the original numerical gates. Whole-model +precision remains a separate validation requirement. +The L20 visual-stage regression is exact for the pinned protected PNG case; +the original full 280-answer gate still fails (177 within 0.01 and three +protected JPEG disagreements). See the recipe for the measured scope and +remaining language/LM/LoRA and JPEG validation gaps. + [`executor.rs`](native/src/executor.rs) runs the vision tower once per request, then the request's prefix up to its last multiple of 64 tokens once, each question's text up to the last multiple of 64 before its first option once, and each option's block with the diff --git a/src/models/omnijev/native/src/executor.rs b/src/models/omnijev/native/src/executor.rs index 21d09b31..2fac1331 100644 --- a/src/models/omnijev/native/src/executor.rs +++ b/src/models/omnijev/native/src/executor.rs @@ -23,6 +23,23 @@ use crate::processing::{IMAGE_TOKEN, PreparedRequest, Processor, Row}; #[path = "../../../../../tests/omnijev/window_graph.rs"] mod window_graph_tests; +/// Vision arithmetic only; language weights/readout and finishing are unchanged. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum VisionNumerics { + Legacy, + Reference, +} +impl std::str::FromStr for VisionNumerics { + type Err = anyhow::Error; + fn from_str(value: &str) -> Result { + match value { + "legacy" => Ok(Self::Legacy), + "reference" => Ok(Self::Reference), + _ => anyhow::bail!("OMNIJEV_VISION_NUMERICS must be legacy or reference"), + } + } +} + pub struct Executor { vision: VisionModel, language: Model, @@ -100,6 +117,20 @@ impl Executor { /// Check the export at `dir`, then load the processor, the vision tower, the /// language model and the heads. pub fn load(dir: &Path, library: &Path) -> Result<(Self, Processor)> { + let numerics = std::env::var("OMNIJEV_VISION_NUMERICS") + .or_else(|error| match error { + std::env::VarError::NotPresent => Ok("legacy".to_owned()), + error => Err(error), + })? + .parse()?; + Self::load_with_vision_numerics(dir, library, numerics) + } + /// An explicit selection for callers that do not use worker environment settings. + pub fn load_with_vision_numerics( + dir: &Path, + library: &Path, + numerics: VisionNumerics, + ) -> Result<(Self, Processor)> { let export = export::load(dir)?; let processor = Processor::load(dir)?; let language = Model::load(dir, library)?; @@ -108,7 +139,10 @@ impl Executor { && language.cfg.hidden == heads::HIDDEN, "export config does not match OmniJev-4B" ); - let vision = VisionModel::load(dir, library)?; + let mut vision = VisionModel::load(dir, library)?; + if numerics == VisionNumerics::Reference { + vision.enable_omnijev_numerics()?; + } let heads = Heads::load(&dir.join("heads.safetensors"))?; Ok(( Self { diff --git a/src/models/qwen3_5/native/src/cuda.rs b/src/models/qwen3_5/native/src/cuda.rs index 409a0e2d..57dfed00 100644 --- a/src/models/qwen3_5/native/src/cuda.rs +++ b/src/models/qwen3_5/native/src/cuda.rs @@ -9,7 +9,7 @@ use std::sync::OnceLock; use anyhow::{Context, Result, bail, ensure}; mod reference; -pub use reference::Reference; +pub use reference::{OmniJevVision, Reference}; /// `CS1_ABI_VERSION` in ops.h. const ABI_VERSION: u32 = 9; @@ -258,6 +258,7 @@ macro_rules! api { _lib: libloading::Library, vision_v2: Option, reference: Option, + omnijev_vision: Option, $(pub $name: unsafe extern "C" fn($($ty),*) $(-> $ret)?,)* } @@ -275,6 +276,13 @@ macro_rules! api { pub fn vision_v2(&self) -> Result<&VisionV2> { self.vision_v2.as_ref().context("27B vision needs additive v2 CUDA symbols; rebuild libqwen3_5_cuda.so") } + pub fn omnijev_vision(&self) -> Result<&OmniJevVision> { + let vision = self.omnijev_vision.as_ref().context("OmniJev-4B requires optional vision CUDA symbols; rebuild libqwen3_5_cuda.so")?; + let reference = self.language_reference()?; + // SAFETY: the capability query does no device work. + ensure!(unsafe { (reference.available)() } == 1, "OmniJev-4B vision requires a cuDNN 9 build; set CUDNN_INCLUDE_DIR and CUDNN_LIB_DIR"); + Ok(vision) + } fn resolve(lib: libloading::Library) -> Result { $( // SAFETY: the signature is the one ops.h declares for this symbol. @@ -288,7 +296,8 @@ macro_rules! api { )* let vision_v2=VisionV2::resolve(&lib); let reference=Reference::resolve(&lib); - Ok(Self { _lib: lib, vision_v2, reference, $($name,)* }) + let omnijev_vision=OmniJevVision::resolve(&lib); + Ok(Self { _lib: lib, vision_v2, reference, omnijev_vision, $($name,)* }) } } }; diff --git a/src/models/qwen3_5/native/src/cuda/reference.rs b/src/models/qwen3_5/native/src/cuda/reference.rs index bd93b388..03c6edd3 100644 --- a/src/models/qwen3_5/native/src/cuda/reference.rs +++ b/src/models/qwen3_5/native/src/cuda/reference.rs @@ -137,3 +137,57 @@ impl Reference { } } } + +/// Optional, separately selected unadapted OmniJev-4B vision operators. +/// Old ABI-9 libraries remain loadable; requesting this group requires a rebuild. +pub struct OmniJevVision { + pub patch_create: unsafe extern "C" fn(c_int, c_int, Stream) -> *mut c_void, + pub norm: unsafe extern "C" fn( + *const c_void, + *const c_void, + *const c_void, + *mut c_void, + c_int, + c_int, + Stream, + ) -> c_int, + pub rope: unsafe extern "C" fn( + *const c_void, + *const f32, + *const f32, + *mut c_void, + *mut c_void, + c_int, + c_int, + c_int, + Stream, + ) -> c_int, + pub attention_workspace_floats: unsafe extern "C" fn(c_int, c_int) -> usize, + pub attention: unsafe extern "C" fn( + *const c_void, + *const c_void, + *const c_void, + *mut c_void, + c_int, + c_int, + c_int, + *mut c_void, + Stream, + ) -> c_int, +} +impl OmniJevVision { + pub(super) fn resolve(lib: &libloading::Library) -> Option { + // SAFETY: these optional signatures match ops.h. Resolve the complete group. + unsafe { + Some(Self { + patch_create: *lib.get(b"cs1_reference_patch_create_v2\0").ok()?, + norm: *lib.get(b"cs1_omnijev_vision_norm\0").ok()?, + rope: *lib.get(b"cs1_omnijev_vision_rope\0").ok()?, + attention_workspace_floats: *lib + .get(b"cs1_omnijev_vision_attention_workspace_floats\0") + .ok()?, + attention: *lib.get(b"cs1_omnijev_vision_attention\0").ok()?, + }) + } + } +} diff --git a/src/models/qwen3_5/native/src/vision/model.rs b/src/models/qwen3_5/native/src/vision/model.rs index e16865c9..c20e96ab 100644 --- a/src/models/qwen3_5/native/src/vision/model.rs +++ b/src/models/qwen3_5/native/src/vision/model.rs @@ -16,7 +16,10 @@ use std::{ path::Path, }; -type Trace<'a> = Option<&'a mut dyn FnMut(&str, &[bf16]) -> Result<()>>; +type Trace<'a> = Option<( + Option, + &'a mut dyn FnMut(&str, &[bf16]) -> Result<()>, +)>; struct OwnedStream(Stream); impl Drop for OwnedStream { @@ -49,6 +52,7 @@ pub struct VisionModel { lora: BTreeMap, gemm: Gemm, reference: Option<&'static cuda::Reference>, + omnijev: Option<&'static cuda::OmniJevVision>, stream: OwnedStream, } #[derive(Clone, Copy)] @@ -171,6 +175,7 @@ impl VisionModel { stream, gemm, reference: None, + omnijev: None, }; for (name, tensor) in base { model.base.insert( @@ -222,6 +227,41 @@ impl VisionModel { self.reference = Some(reference); Ok(()) } + /// Select the unadapted OmniJev-4B BF16 vision arithmetic before any forward. + /// This leaves language numerics, the generic path and JEMM-27B unchanged. + pub fn enable_omnijev_numerics(&mut self) -> Result<()> { + ensure!( + self.config.hidden_size == 1024 && self.lora.is_empty(), + "OmniJev reference vision requires unadapted 4B vision" + ); + ensure!( + self.scratch.is_empty() && self.batch_scratch.is_empty() && self.transient.is_none(), + "select OmniJev reference vision before the first forward" + ); + ensure!( + self.cache_limits.is_none(), + "OmniJev reference vision uses the generic FIFO cache; bounded cache cannot account for opaque cuDNN workspace" + ); + if self.omnijev.is_some() { + return Ok(()); + } + let omnijev = api().omnijev_vision()?; + let reference = api().language_reference()?; + cuda::set_device(0)?; + self.synchronize()?; + // Torch's vision addmm uses a 1 MiB Lt heuristic budget. A larger budget + // changes split-K selection/rounding in the merger even on the same GPU. + // SAFETY: no plans, buffers or captures exist and the old handle is drained. + let gemm = Gemm(unsafe { (reference.gemm_create)(1 << 20) }); + ensure!( + !gemm.0.is_null(), + "cannot create OmniJev reference vision GEMM handle" + ); + self.gemm = gemm; + self.reference = Some(reference); + self.omnijev = Some(omnijev); + Ok(()) + } fn upload(&self, bytes: &[u8]) -> Result { let buffer = DeviceBuffer::new(bytes.len())?; unsafe { @@ -233,11 +273,13 @@ impl VisionModel { self.base[name].at(0) } fn norm(&self, name: &str, x: &DeviceBuffer, y: &DeviceBuffer, rows: usize) -> Result<()> { + let norm = self + .reference + .map_or(api().cs1_vision_norm, |r| r.vision_norm); + let norm = self.omnijev.map_or(norm, |r| r.norm); unsafe { check( - (self - .reference - .map_or(api().cs1_vision_norm, |r| r.vision_norm))( + norm( x.at(0), self.weight(&format!("{name}.weight")), self.weight(&format!("{name}.bias")), @@ -365,11 +407,27 @@ impl VisionModel { x: &DeviceBuffer, n: usize, ) -> Result<()> { - if let Some(callback) = callback.as_mut() { + if let Some((_, callback)) = callback.as_mut() { callback(name, &self.read(x, n)?)?; } Ok(()) } + fn trace_block( + &self, + callback: &mut Trace<'_>, + block: usize, + name: &str, + x: &DeviceBuffer, + n: usize, + ) -> Result<()> { + if callback + .as_ref() + .is_some_and(|(selected, _)| *selected == Some(block)) + { + self.trace(callback, &format!("blocks.{block}.{name}"), x, n)?; + } + Ok(()) + } pub fn synchronize(&self) -> Result<()> { cuda::set_device(0)?; cuda::synchronize(self.stream.0) @@ -485,7 +543,21 @@ impl VisionModel { image: &ProcessedImage, mut callback: impl FnMut(&str, &[bf16]) -> Result<()>, ) -> Result> { - self.run(image, Some(&mut callback)) + self.run(image, Some((None, &mut callback))) + } + /// The usual stages plus operator boundaries of one selected block. Eager only; + /// callback errors drain queued work before temporary storage can retire. + pub fn forward_with_block_trace( + &mut self, + image: &ProcessedImage, + block: usize, + mut callback: impl FnMut(&str, &[bf16]) -> Result<()>, + ) -> Result> { + ensure!( + block < self.config.depth, + "vision trace block is out of range" + ); + self.run(image, Some((Some(block), &mut callback))) } /// Synchronized resident merger output; the borrow prevents cache mutation/retirement. pub fn forward_device(&mut self, image: &ProcessedImage) -> Result> { @@ -515,7 +587,7 @@ impl VisionModel { &self.scratch_at(index).buffers.out, n / 4 * self.config.out_hidden_size, )?; - if let Some(callback) = callback.as_mut() { + if let Some((_, callback)) = callback.as_mut() { callback("merger.output", &output)?; } else if let (true, ScratchLocation::Retained(index)) = (capture, index) { self.capture_current(index); @@ -789,9 +861,40 @@ impl VisionModel { for i in 0..self.config.depth { let p = format!("blocks.{i}"); self.norm(&format!("{p}.norm1"), x, norm, n)?; + self.trace_block(callback, i, "norm1", norm, n * h)?; self.linear(&format!("{p}.attn.qkv"), norm, qkv, n, h * 3, h, w)?; + self.trace_block(callback, i, "attn.qkv", qkv, n * h * 3)?; unsafe { - if h == 1024 { + if let Some(omnijev) = self.omnijev { + check( + (omnijev.rope)( + qkv.at(0), + co.at(0).cast(), + si.at(0).cast(), + q.at(0), + k.at(0), + n as i32, + h as i32, + self.config.head_dim() as i32, + self.stream.0, + ), + "OmniJev vision rotary", + )?; + check( + (omnijev.attention)( + q.at(0), + k.at(0), + qkv.at(h * 2 * 2), + attn.at(0), + n as i32, + self.config.num_heads as i32, + self.config.head_dim() as i32, + attention_workspace.at(0), + self.stream.0, + ), + "OmniJev vision attention", + )?; + } else if h == 1024 { check( (api().cs1_vision_rope)( qkv.at(0), @@ -851,14 +954,20 @@ impl VisionModel { } } } + self.trace_block(callback, i, "attn.q", q, n * h)?; + self.trace_block(callback, i, "attn.k", k, n * h)?; + self.trace_block(callback, i, "attn.output", attn, n * h)?; self.linear(&format!("{p}.attn.proj"), attn, delta, n, h, h, w)?; + self.trace_block(callback, i, "attn.proj", delta, n * h)?; unsafe { check( (api().cs1_vision_add)(x.at(0), delta.at(0), n * h, self.stream.0), "vision attention residual", )?; } + self.trace_block(callback, i, "attn.residual", x, n * h)?; self.norm(&format!("{p}.norm2"), x, norm, n)?; + self.trace_block(callback, i, "norm2", norm, n * h)?; self.linear( &format!("{p}.mlp.linear_fc1"), norm, @@ -868,12 +977,14 @@ impl VisionModel { h, w, )?; + self.trace_block(callback, i, "mlp.linear_fc1", mlp, n * intermediate)?; unsafe { check( (api().cs1_vision_gelu)(mlp.at(0), n * intermediate, 0, self.stream.0), "vision tanh GELU", )?; } + self.trace_block(callback, i, "mlp.activation", mlp, n * intermediate)?; self.linear( &format!("{p}.mlp.linear_fc2"), mlp, @@ -883,6 +994,7 @@ impl VisionModel { intermediate, w, )?; + self.trace_block(callback, i, "mlp.linear_fc2", delta, n * h)?; unsafe { check( (api().cs1_vision_add)(x.at(0), delta.at(0), n * h, self.stream.0), @@ -1044,7 +1156,17 @@ impl Buffers { ); let patch = if let Some(reference) = model.reference { // SAFETY: descriptors/workspace are allocated outside capture for validated rows. - let plan = PatchPlan(unsafe { (reference.patch_create)(n as i32, model.stream.0) }); + let plan = PatchPlan(unsafe { + if let Some(omnijev) = model.omnijev { + (omnijev.patch_create)( + n as i32, + model.config.hidden_size as i32, + model.stream.0, + ) + } else { + (reference.patch_create)(n as i32, model.stream.0) + } + }); ensure!( !plan.0.is_null(), "cannot prepare cuDNN reference patch convolution" @@ -1053,7 +1175,17 @@ impl Buffers { } else { None }; - sizes[8] = if let Some(reference) = model.reference { + sizes[8] = if let Some(omnijev) = model.omnijev { + // SAFETY: this sizing query uses validated bounded single-image geometry. + let floats = unsafe { + (omnijev.attention_workspace_floats)( + attention_rows as i32, + model.config.num_heads as i32, + ) + }; + ensure!(floats > 0, "cannot size OmniJev vision attention workspace"); + floats * 4 + } else if let Some(reference) = model.reference { let floats = unsafe { (reference.vision_attention_workspace_floats)( attention_rows as i32, @@ -1134,3 +1266,7 @@ mod vision_tests; #[cfg(test)] #[path = "../../../../../../tests/qwen3_5/vision_batch.rs"] mod batch_tests; + +#[cfg(test)] +#[path = "../../../../../../tests/qwen3_5/vision_omnijev.rs"] +mod omnijev_tests; diff --git a/tests/omnijev/window_graph.rs b/tests/omnijev/window_graph.rs index 625feae9..d3ce8f1f 100644 --- a/tests/omnijev/window_graph.rs +++ b/tests/omnijev/window_graph.rs @@ -4,6 +4,21 @@ use super::*; #[path = "common.rs"] mod common; +#[test] +fn vision_profile_is_explicit_and_rejects_unknown_settings() { + assert_eq!( + "legacy".parse::().unwrap(), + VisionNumerics::Legacy + ); + assert_eq!( + "reference".parse::().unwrap(), + VisionNumerics::Reference + ); + for unknown in ["", "1", "jemm", "Reference", "reference "] { + assert!(unknown.parse::().is_err(), "{unknown:?}"); + } +} + // Replace the model only after dropping its prefix states and the old model. // This selects both modes without mutating process-wide environment variables. fn language_mode(executor: Executor, dir: &Path, library: &Path, graph: bool) -> Executor { diff --git a/tests/qwen3_5/README.md b/tests/qwen3_5/README.md index cb004468..89f3e9c4 100644 --- a/tests/qwen3_5/README.md +++ b/tests/qwen3_5/README.md @@ -95,3 +95,51 @@ full-checkpoint replay needs its own frozen inputs, artifact identity and model acceptance. Standard workspace checks compile these tests but leave GPU cases ignored; `graph_loading` checks missing capability, unsupported graph/adapter profiles and input rejection without a GPU. + +## OmniJev-4B reference vision + +The optional profile has two independent checks, run on the reserved GPU with +`torch==2.14.0`, `transformers==5.17.0` and `peft==0.19.1` in the documented +OmniJev reference environment. The first needs no model weights: + +```sh +CUA_S1_CUDA_LIB=/absolute/backend/libqwen3_5_cuda.so \ + python -m unittest discover -s tests/qwen3_5 -p test_vision_omnijev.py -v +``` + +It compares LayerNorm and strided-V noncausal attention with Torch BF16 exactly, +including short grids, 64/128-key tile tails and the 4608-patch limit. Invalid +dimensions are rejected before dereferencing device pointers; the existing 27B +entry still rejects the 4B layout. +The cuDNN build also checks that the original patch constructor retains its +1152-wide layout. Optionally set `QWEN_LEGACY_CUDA_LIB` to an immutable pre-change +library to compare existing 4B and JEMM-27B norm/attention outputs bit for bit. + +For the real-weight regression, capture the official MSO1 visual output and its +FP32 `pixel_values` in the pinned environment, preserving the original image, +grid, source/checkpoint hashes and complete request. The generator uses the +unchanged exported visual weights, and refuses its oracle unless the complete +HF visual output equals that actual MSO1 capture bit for bit. Inputs and all +40 expected stages stay in the supplied external directory: + +```sh +python tests/qwen3_5/vision_omnijev_reference.py \ + --base /absolute/qwen3.5-4b --export /absolute/omnijev-export \ + --pixels /absolute/official-pixels.f32 --grid 1 36 64 \ + --mso1-output /absolute/official-vision.bf16 --out /absolute/new-stage-oracle +OMNIJEV_EXPORT=/absolute/omnijev-export \ +CUA_S1_CUDA_LIB=/absolute/backend/libqwen3_5_cuda.so \ +OMNIJEV_VISION_ORACLE=/absolute/new-stage-oracle \ + cargo test --release --locked -p omni-qwen3-5-native --lib \ + omnijev_reference_matches_official_vision_stages_and_graph_replay \ + -- --ignored --test-threads=1 +``` + +The test requires exact patch, position, all 24 blocks, merger and block0 +operator outputs. It also checks profile/adapter/cache/late-selection errors, +actual vision Graph capture/replay, resident-output readback and recovery after +a trace callback error. The existing `forward_with_trace` keeps its 29 events; +`forward_with_block_trace` adds one selected block's operator boundaries and +never captures or replays a Graph while tracing. Ordinary CPU CI compiles and +skips the device case. This vision oracle does not replace the original full +280-answer probability and protected-decision gates, or language/LoRA parity. diff --git a/tests/qwen3_5/test_vision_omnijev.py b/tests/qwen3_5/test_vision_omnijev.py new file mode 100644 index 00000000..1d60f4ab --- /dev/null +++ b/tests/qwen3_5/test_vision_omnijev.py @@ -0,0 +1,135 @@ +"""Opt-in head-64 reference operators, without checkpoint weights. + +CUA_S1_CUDA_LIB=/absolute/libqwen3_5_cuda.so python -m unittest discover \ + -s tests/qwen3_5 -p test_vision_omnijev.py -v +Uses the pinned OmniJev Torch 2.14.0 CUDA reference environment. +""" +import ctypes +import os +import unittest + + +@unittest.skipUnless(os.environ.get("CUA_S1_CUDA_LIB"), "needs CUDA and CUA_S1_CUDA_LIB") +class OmniJevVisionOperators(unittest.TestCase): + @classmethod + def setUpClass(cls): + import torch + + cls.torch = torch + if torch.__version__.split("+")[0] != "2.14.0": + raise unittest.SkipTest("requires pinned torch 2.14.0") + torch.manual_seed(114) + cls.library = ctypes.CDLL(os.environ["CUA_S1_CUDA_LIB"]) + cls.stream = torch.cuda.current_stream().cuda_stream + + def bind(self, name, arguments, result=ctypes.c_int): + operation = getattr(self.library, name) + operation.argtypes = arguments + operation.restype = result + return operation + + def test_attention_matches_sdpa_for_short_tail_and_original_grid(self): + torch = self.torch + pointer, integer = ctypes.c_void_p, ctypes.c_int + operation = self.bind("cs1_omnijev_vision_attention", [pointer] * 4 + [integer] * 3 + [pointer] * 2) + size = self.bind("cs1_omnijev_vision_attention_workspace_floats", [integer, integer], ctypes.c_size_t) + for rows in [4, 16, 60, 64, 68, 124, 128, 132, 256, 2304, 4608]: + with self.subTest(rows=rows): + qkv = torch.randn(rows, 3, 16, 64, device="cuda", dtype=torch.bfloat16) + q, k = qkv[:, 0].contiguous(), qkv[:, 1].contiguous() + actual = torch.empty_like(q) + workspace = torch.empty(size(rows, 16), device="cuda", dtype=torch.float32) + expected = torch.nn.functional.scaled_dot_product_attention( + q.transpose(0, 1).unsqueeze(0), k.transpose(0, 1).unsqueeze(0), + qkv[:, 2].transpose(0, 1).unsqueeze(0), scale=0.125, + ).squeeze(0).transpose(0, 1) + self.assertEqual(operation(q.data_ptr(), k.data_ptr(), qkv.data_ptr() + 4096, actual.data_ptr(), rows, 16, 64, workspace.data_ptr(), self.stream), 0) + self.assertTrue(torch.equal(actual, expected), f"{rows}: {(actual != expected).sum().item()} BF16 values differ") + + def test_norm_matches_framework_vectorized_welford(self): + torch = self.torch + pointer, integer = ctypes.c_void_p, ctypes.c_int + operation = self.bind("cs1_omnijev_vision_norm", [pointer] * 4 + [integer, integer, pointer]) + for rows in [4, 32, 2304]: + with self.subTest(rows=rows): + x = torch.randn(rows, 1024, device="cuda", dtype=torch.bfloat16) + weight = torch.randn(1024, device="cuda", dtype=torch.bfloat16) + bias = torch.randn_like(weight) + actual = torch.empty_like(x) + expected = torch.nn.functional.layer_norm(x, (1024,), weight, bias, eps=1e-6) + self.assertEqual(operation(x.data_ptr(), weight.data_ptr(), bias.data_ptr(), actual.data_ptr(), rows, 1024, self.stream), 0) + self.assertTrue(torch.equal(actual, expected)) + + def test_layout_guards_keep_27b_separate(self): + pointer, integer = ctypes.c_void_p, ctypes.c_int + for name in ["cs1_omnijev_vision_norm", "cs1_reference_vision_norm"]: + operation = self.bind(name, [pointer] * 4 + [integer, integer, pointer]) + wrong_width = 1152 if "omnijev" in name else 1024 + self.assertNotEqual(operation(None, None, None, None, 1, wrong_width, self.stream), 0) + attention = self.bind("cs1_omnijev_vision_attention", [pointer] * 4 + [integer] * 3 + [pointer] * 2) + for rows, heads, dimension in [(0, 16, 64), (4609, 16, 64), (4, 8, 64), (4, 16, 72)]: + self.assertNotEqual(attention(None, None, None, None, rows, heads, dimension, None, self.stream), 0) + create = self.bind("cs1_reference_patch_create_v2", [integer, integer, pointer], pointer) + self.assertIsNone(create(4, 512, self.stream)) + + @unittest.skipUnless(os.environ.get("QWEN_LEGACY_CUDA_LIB"), "needs an immutable pre-change CUDA library") + def test_existing_4b_and_27b_operators_keep_their_bits(self): + torch = self.torch + pointer, integer = ctypes.c_void_p, ctypes.c_int + old = ctypes.CDLL(os.environ["QWEN_LEGACY_CUDA_LIB"]) + + def pair(name, arguments, result=integer): + new = self.bind(name, arguments, result) + previous = getattr(old, name) + previous.argtypes, previous.restype = arguments, result + return previous, new + + for name, width in [("cs1_vision_norm", 1024), ("cs1_reference_vision_norm", 1152)]: + x = torch.randn(132, width, device="cuda", dtype=torch.bfloat16) + w = torch.randn(width, device="cuda", dtype=torch.bfloat16) + b = torch.randn_like(w) + outputs = [torch.empty_like(x), torch.empty_like(x)] + for operation, output in zip(pair(name, [pointer] * 4 + [integer, integer, pointer]), outputs): + self.assertEqual(operation(x.data_ptr(), w.data_ptr(), b.data_ptr(), output.data_ptr(), 132, width, self.stream), 0) + self.assertTrue(torch.equal(*outputs), name) + + for dimension in [64, 72]: + qkv = torch.randn(132, 3, 16, dimension, device="cuda", dtype=torch.bfloat16) + q, k = qkv[:, 0].contiguous(), qkv[:, 1].contiguous() + outputs = [torch.empty_like(q), torch.empty_like(q)] + if dimension == 64: + operations = pair("cs1_vision_attention", [pointer] * 4 + [integer, pointer]) + for operation, output in zip(operations, outputs): + self.assertEqual(operation(q.data_ptr(), k.data_ptr(), qkv[:, 2].data_ptr(), output.data_ptr(), 132, self.stream), 0) + else: + operations = pair("cs1_reference_vision_attention", [pointer] * 4 + [integer] * 3 + [pointer] * 2) + sizes = pair("cs1_reference_vision_attention_workspace_floats", [integer, integer], ctypes.c_size_t) + for operation, size, output in zip(operations, sizes, outputs): + workspace = torch.empty(132 * 16 * 80 * 2 + size(132, 16), device="cuda", dtype=torch.float32) + self.assertEqual(operation(q.data_ptr(), k.data_ptr(), qkv[:, 2].data_ptr(), output.data_ptr(), 132, 16, 72, workspace.data_ptr(), self.stream), 0) + self.assertTrue(torch.equal(*outputs), f"legacy attention head dimension {dimension}") + + def test_original_patch_constructor_keeps_27b_layout(self): + torch = self.torch + pointer, integer = ctypes.c_void_p, ctypes.c_int + old_create = self.bind("cs1_reference_patch_create", [integer, pointer], pointer) + new_create = self.bind("cs1_reference_patch_create_v2", [integer, integer, pointer], pointer) + execute = self.bind("cs1_reference_patch", [pointer] * 4) + destroy = self.bind("cs1_reference_patch_destroy", [pointer], None) + x = torch.randn(4, 1536, device="cuda", dtype=torch.bfloat16) + w = torch.randn(1152, 1536, device="cuda", dtype=torch.bfloat16) + outputs = [torch.empty(4, 1152, device="cuda", dtype=torch.bfloat16) for _ in range(2)] + plans = [old_create(4, self.stream), new_create(4, 1152, self.stream)] + self.assertTrue(all(plans), "requires the cuDNN 9 build") + try: + for plan, output in zip(plans, outputs): + self.assertEqual(execute(plan, x.data_ptr(), w.data_ptr(), output.data_ptr()), 0) + self.assertTrue(torch.equal(*outputs)) + finally: + torch.cuda.synchronize() + for plan in plans: + destroy(plan) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/qwen3_5/vision_batch.rs b/tests/qwen3_5/vision_batch.rs index a157bf39..a370a25d 100644 --- a/tests/qwen3_5/vision_batch.rs +++ b/tests/qwen3_5/vision_batch.rs @@ -218,12 +218,20 @@ fn cpu_batch_uses_aggregate_gemms_and_image_local_attention() { batch_scratch: VecDeque::new(), graph_enabled: false, reference: None, + omnijev: None, base, lora: BTreeMap::new(), gemm: Gemm(unsafe { (api().cs1_gemm_create)(32 << 20) }), stream: OwnedStream(cuda::new_stream().unwrap()), }; let images = [image([1, 2, 2]), image([1, 2, 4])]; + assert!( + model + .enable_omnijev_numerics() + .unwrap_err() + .to_string() + .contains("unadapted 4B") + ); let result = model.forward_images(&images).unwrap(); assert_eq!(result.len(), 3 * 5120); for index in 0..unsafe { count() } { @@ -440,6 +448,18 @@ fn cpu_batch_uses_aggregate_gemms_and_image_local_attention() { assert!(model.transient.is_none()); assert_eq!(unsafe { live_bytes() }, model.retained_bytes()); assert_eq!(unsafe { unsafe_frees() }, 0); + model.synchronize().unwrap(); + model.scratch.clear(); + model.lora.clear(); + model.cache_limits = None; + assert!( + model + .enable_omnijev_numerics() + .unwrap_err() + .to_string() + .contains("optional vision CUDA symbols") + ); + assert!(model.reference.is_none() && model.omnijev.is_none()); drop(model); assert_eq!(unsafe { live() }, 0); drop(metrics); diff --git a/tests/qwen3_5/vision_omnijev.rs b/tests/qwen3_5/vision_omnijev.rs new file mode 100644 index 00000000..32dbedab --- /dev/null +++ b/tests/qwen3_5/vision_omnijev.rs @@ -0,0 +1,136 @@ +//! Exact real-weight stage regression; external oracle generated in the pinned env. +use super::*; + +#[test] +#[ignore = "needs GPU, OMNIJEV_EXPORT, CUA_S1_CUDA_LIB and OMNIJEV_VISION_ORACLE"] +fn omnijev_reference_matches_official_vision_stages_and_graph_replay() { + let path = |name| std::path::PathBuf::from(std::env::var_os(name).expect(name)); + let oracle = path("OMNIJEV_VISION_ORACLE"); + let fixture: serde_json::Value = + serde_json::from_slice(&std::fs::read(oracle.join("reference.json")).unwrap()).unwrap(); + assert_eq!(fixture["schema"], "omnijev-4b-vision-reference/1"); + assert_eq!(fixture["mso1_output_exact"], true); + let grid: [usize; 3] = serde_json::from_value(fixture["grid"].clone()).unwrap(); + let bytes = std::fs::read(oracle.join("pixels.f32")).unwrap(); + assert_eq!(bytes.len(), grid[1] * grid[2] * 1536 * 4); + let image = ProcessedImage { + pixel_values: bytes + .as_chunks::<4>() + .0 + .iter() + .map(|&b| f32::from_le_bytes(b)) + .collect(), + image_grid_thw: grid, + resized_height: grid[1] * 16, + resized_width: grid[2] * 16, + }; + let mut model = VisionModel::load(path("OMNIJEV_EXPORT"), &path("CUA_S1_CUDA_LIB")).unwrap(); + assert!( + model.enable_reference_numerics().is_err(), + "4B must not silently select JEMM" + ); + model.cache_limits = Some(CacheLimits { + entries: 1, + bytes: usize::MAX, + }); + assert!( + model + .enable_omnijev_numerics() + .unwrap_err() + .to_string() + .contains("bounded cache") + ); + model.cache_limits = None; + model + .lora + .insert("rejected_adapter".into(), DeviceBuffer::new(0).unwrap()); + assert!( + model + .enable_omnijev_numerics() + .unwrap_err() + .to_string() + .contains("unadapted 4B") + ); + model.lora.clear(); + model.enable_omnijev_numerics().unwrap(); + model.enable_omnijev_numerics().unwrap(); + let expected: BTreeMap> = fixture["stages"] + .as_array() + .unwrap() + .iter() + .map(|stage| { + let name = stage["name"].as_str().unwrap().to_owned(); + let bytes = std::fs::read(oracle.join(format!("{name}.bf16"))).unwrap(); + let shape: Vec = serde_json::from_value(stage["shape"].clone()).unwrap(); + assert_eq!(shape.iter().product::() * 2, bytes.len(), "{name}"); + (name, bytes) + }) + .collect(); + let mut seen = std::collections::BTreeSet::new(); + let result = model + .forward_with_block_trace(&image, 0, |name, values| { + let actual: Vec<_> = values.iter().flat_map(|v| v.to_le_bytes()).collect(); + let wanted = &expected[name]; + let different = actual + .as_chunks::<2>() + .0 + .iter() + .zip(wanted.as_chunks::<2>().0.iter()) + .filter(|(a, b)| a != b) + .count(); + assert_eq!(different, 0, "{name}: BF16 values differ"); + assert_eq!(actual.len(), wanted.len(), "{name}"); + seen.insert(name.to_owned()); + Ok(()) + }) + .unwrap(); + assert_eq!(seen, expected.keys().cloned().collect()); + assert!( + model + .enable_omnijev_numerics() + .unwrap_err() + .to_string() + .contains("before the first forward") + ); + let bits = |values: &[bf16]| values.iter().map(|v| v.to_bits()).collect::>(); + model.graph_enabled = true; + assert_eq!(bits(&model.forward(&image).unwrap()), bits(&result)); + assert!( + model.scratch[0].graph.is_some(), + "capture must actually succeed" + ); + assert_eq!(bits(&model.forward(&image).unwrap()), bits(&result)); + assert_eq!( + bits(&model.forward_device(&image).unwrap().download().unwrap()), + bits(&result) + ); + let mut changed = ProcessedImage { + pixel_values: image.pixel_values.clone(), + image_grid_thw: grid, + resized_height: image.resized_height, + resized_width: image.resized_width, + }; + for pixel in changed.pixel_values.iter_mut().take(1536) { + *pixel += 0.125; + } + model.graph_enabled = false; + let changed_eager = model.forward(&changed).unwrap(); + assert_ne!(bits(&changed_eager), bits(&result)); + model.graph_enabled = true; + assert_eq!( + bits(&model.forward(&changed).unwrap()), + bits(&changed_eager) + ); + assert_eq!(bits(&model.forward(&image).unwrap()), bits(&result)); + assert!( + model + .forward_with_block_trace(&image, model.config.depth, |_, _| Ok(())) + .is_err() + ); + assert!( + model + .forward_with_block_trace(&image, 0, |_, _| anyhow::bail!("trace error")) + .is_err() + ); + assert_eq!(bits(&model.forward(&image).unwrap()), bits(&result)); +} diff --git a/tests/qwen3_5/vision_omnijev_reference.py b/tests/qwen3_5/vision_omnijev_reference.py new file mode 100644 index 00000000..395a0ee8 --- /dev/null +++ b/tests/qwen3_5/vision_omnijev_reference.py @@ -0,0 +1,108 @@ +"""Generate an external, exact BF16 vision-stage oracle in the pinned reference env. + +No weights or run artifacts are committed. See tests/qwen3_5/README.md for use. +The complete HF visual output must equal an actual official MSO1 capture before +its intermediate tensors are accepted as an oracle. +""" +import argparse +import hashlib +import json +from pathlib import Path + +import peft +import torch +import transformers +from safetensors.torch import load_file +from transformers import AutoConfig +from transformers.models.qwen3_5 import modeling_qwen3_5 as modeling + + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base", type=Path, required=True) + parser.add_argument("--export", type=Path, required=True) + parser.add_argument("--pixels", type=Path, required=True, help="official FP32 pixel_values, flat little-endian") + parser.add_argument("--grid", type=int, nargs=3, required=True) + parser.add_argument("--mso1-output", type=Path, required=True, help="actual official visual BF16 capture") + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + assert torch.__version__.split("+")[0] == "2.14.0" + assert transformers.__version__ == "5.17.0" and peft.__version__ == "0.19.1" + assert args.grid[0] == 1 + args.out.mkdir(parents=True, exist_ok=False) + torch.backends.cuda.matmul.allow_tf32 = False + config = AutoConfig.from_pretrained(args.base, local_files_only=True).vision_config + assert config.hidden_size == 1024 and config.depth == 24 + config._attn_implementation = "sdpa" + previous = torch.get_default_dtype() + torch.set_default_dtype(torch.bfloat16) + try: + model = modeling.Qwen3_5VisionModel(config) + finally: + torch.set_default_dtype(previous) + assert model.rotary_pos_emb.inv_freq.dtype == torch.float32 + model.load_state_dict({k.removeprefix("model.visual."): v for k, v in load_file(args.export / "vision.safetensors").items()}, strict=True) + model = model.cuda().eval() + stages = [] + + def record(name, tensor): + tensor = tensor.detach().cpu().contiguous() + assert tensor.dtype == torch.bfloat16 + path = args.out / (name + ".bf16") + path.write_bytes(tensor.view(torch.uint16).numpy().tobytes()) + stages.append({"name": name, "shape": list(tensor.shape), "sha256": digest(path)}) + + model.patch_embed.register_forward_hook(lambda mod, inputs, out: record("patch_embed", out)) + model.blocks[0].register_forward_pre_hook(lambda mod, inputs: record("position", inputs[0])) + for index, block in enumerate(model.blocks): + block.register_forward_hook(lambda mod, inputs, out, index=index: record(f"blocks.{index}", out)) + for name in ["norm1", "attn.qkv", "attn.proj", "norm2", "mlp.linear_fc1", "mlp.act_fn", "mlp.linear_fc2"]: + target = "mlp.activation" if name == "mlp.act_fn" else name + model.blocks[0].get_submodule(name).register_forward_hook(lambda mod, inputs, out, target=target: record("blocks.0." + target, out)) + model.blocks[0].attn.proj.register_forward_pre_hook(lambda mod, inputs: record("blocks.0.attn.output", inputs[0])) + model.blocks[0].norm2.register_forward_pre_hook(lambda mod, inputs: record("blocks.0.attn.residual", inputs[0])) + model.merger.norm.register_forward_hook(lambda mod, inputs, out: record("merger.norm", out)) + model.merger.linear_fc1.register_forward_hook(lambda mod, inputs, out: record("merger.linear_fc1", out)) + original_rope = modeling.apply_rotary_pos_emb_vision + seen_rope = False + + def rope(*inputs): + nonlocal seen_rope + result = original_rope(*inputs) + if not seen_rope: + record("blocks.0.attn.q", result[0]) + record("blocks.0.attn.k", result[1]) + seen_rope = True + return result + + pixels_bytes = args.pixels.read_bytes() + pixels = torch.frombuffer(bytearray(pixels_bytes), dtype=torch.float32).reshape(-1, 1536) + assert pixels.shape[0] == args.grid[1] * args.grid[2] and torch.isfinite(pixels).all() + (args.out / "pixels.f32").write_bytes(pixels_bytes) + modeling.apply_rotary_pos_emb_vision = rope + try: + with torch.inference_mode(): + result = model(pixels.cuda().bfloat16(), grid_thw=torch.tensor([args.grid], device="cuda"), return_dict=True).pooler_output + finally: + modeling.apply_rotary_pos_emb_vision = original_rope + actual = torch.frombuffer(bytearray(args.mso1_output.read_bytes()), dtype=torch.bfloat16).reshape_as(result) + assert torch.equal(result.cpu(), actual), "reject oracle: HF visual differs from actual official MSO1" + record("merger.output", result) + manifest = { + "schema": "omnijev-4b-vision-reference/1", "grid": args.grid, + "torch": torch.__version__, "transformers": transformers.__version__, "peft": peft.__version__, + "export_manifest_sha256": digest(args.export / "omnijev_export.json"), + "vision_weights_sha256": digest(args.export / "vision.safetensors"), + "pixels_sha256": digest(args.pixels), "mso1_output_sha256": digest(args.mso1_output), + "mso1_output_exact": True, "stages": stages, + } + (args.out / "reference.json").write_text(json.dumps(manifest, indent=2) + "\n") + print(json.dumps({"mso1_output_exact": True, "stages": len(stages), "output": str(args.out)})) + + +if __name__ == "__main__": + main() From 72af54fbca76fe34fc525cc4e5d49a7bbe89bd36 Mon Sep 17 00:00:00 2001 From: psv666 <2693925048@qq.com> Date: Fri, 20 Nov 2026 13:40:49 +0800 Subject: [PATCH 2/4] omnijev: add eager unmerged reference Score readout --- docs/architecture.md | 9 ++ recipe/omnijev/README.md | 58 +++++++++- recipe/omnijev/export.py | 30 ++++- recipe/omnijev/native.md | 9 +- src/models/omnijev/README.md | 12 ++ src/models/omnijev/native/src/executor.rs | 127 ++++++++++++++++++++-- src/models/omnijev/native/src/export.rs | 95 +++++++++++++++- src/models/qwen3_5/native/src/model.rs | 49 +++++++++ tests/omnijev/export.rs | 31 ++++++ tests/omnijev/window_graph.rs | 2 + tests/omnijev/worker.rs | 94 +++++++++++++++- tests/qwen3_5/readout.rs | 55 +++++++++- 12 files changed, 548 insertions(+), 23 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index a08ec776..70837b92 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -144,6 +144,15 @@ call. The Cua-S1 text worker uses it ([#188](https://github.com/InferMatrixLab/system1-omni/issues/188)). +OmniJev's separate unmerged one-shot Score library profile uses +`forward_multimodal_reference_readout`: one complete unpadded multimodal row, +the existing reference norm/projection/attention and FP32 GDN layer loop, then +selected final-norm hidden rows in caller order. It synchronizes before returning +or reusing scratch, including error paths. It creates no prefix state and rejects +Graph and LM targets; the fixed/window/reuse guards above remain in effect. +This is a bounded library capability, with full worker/all-type parity unverified. +See the [OmniJev preparation recipe](../recipe/omnijev/README.md#separate-unmerged-reference-artifact). + ## Native worker boundaries The native workers separate `processing.rs` from `executor.rs`; their worker diff --git a/recipe/omnijev/README.md b/recipe/omnijev/README.md index 0ec8aa2d..b7f864e0 100644 --- a/recipe/omnijev/README.md +++ b/recipe/omnijev/README.md @@ -57,12 +57,68 @@ under a minute. ## Fixtures and tests +### Separate unmerged reference artifact + +For the eager one-shot Score **library API**, prepare a separate directory: + +```bash +.venv-omnijev/bin/python recipe/omnijev/export.py --base qwen3.5-4b \ + --checkpoint omnijev-v1.1 --out omnijev-unmerged-reference \ + --language unmerged-reference +``` + +This writes `omnijev-unmerged-reference/1`: the original BF16 base language +weights, resized tied embedding and saved special rows, plus all 400 separate +FP32 rank-32 adapter tensors at alpha 64 / scale 2. Vision, heads, tokenizer and +calibration retain the same identity. It does not merge or overwrite an existing +artifact. Both loaders reject the other format; the merged worker does not serve +this artifact. + +Use `Executor::load_one_shot_reference` and `score_one_shot(&prepared, index)` +after preparing the **complete original request**. Keep every question in the +processor input, since selecting a question before compilation changes its +longest common prefix. The returned selected hidden rows, ordinal probabilities +and calibrated answer support boundary validation. This API supports one Score +question at a time; whole-request execution, Choice/Noul, LM targets, prefix +reuse and Graphs are rejected. Set `CUA_S1_GRAPH=0`. There is no worker switch +for this incomplete all-type profile. + +The root real-weight checks are opt-in and must run serially on a reserved GPU: + +```bash +CUA_S1_GRAPH=0 CUA_S1_CUDA_LIB=/absolute/libqwen3_5_cuda.so \ +QWEN3_5_CHECKPOINT=/absolute/omnijev-unmerged-reference \ +QWEN3_5_ADAPTER=/absolute/omnijev-unmerged-reference/adapter_model.safetensors \ +cargo test --release --locked -p omni-qwen3-5-native --test readout \ + reference_selected_rows_preserve_full_row_and_recover -- --ignored --exact --test-threads=1 +``` + +`tests/omnijev/worker.rs::unmerged_reference_original_q2_gate` consumes the original +12-question PNG request and an official full-request oracle supplied through +`OMNIJEV_ORIGINAL_REQUEST` and `OMNIJEV_REFERENCE_ORACLE` (the observer JSON's +`official_full_answer.q2`). Also set `OMNIJEV_REFERENCE_EXPORT` and +`OMNIJEV_CUDA_LIB` to absolute paths. The test checks prefix 581, input usage +1214, mode rejection, repeated hidden/head outputs, the Score decision, and every +returned probability/confidence at the original 0.01 gate. Optional +`OMNIJEV_REFERENCE_RESULT` saves its selected hidden/head output outside the repo. +Passing this bounded check does not establish all-type/full-280, JPEG or HTTP +parity; those require separate complete execution evidence. + +On one L20, the original full-context PNG `video.png/q2` check passes: selected +`zq` and three `u` rows exactly match the pinned official unmerged plain-row +control; native confidence is 0.5640 against the normative original branch's +0.5613, within the unchanged 0.01 gate. The branch oracle remains authoritative; +the plain-row control only verifies this library path. This coverage is limited +to that observed Score case and the selected-row state/recovery tests. + +### Ordinary CPU fixtures + ```bash .venv-omnijev/bin/python tests/omnijev/generate_fixtures.py \ --reference OmniJev --checkpoint omnijev-v1.1 --base qwen3.5-4b --out tests/omnijev/data cargo test --locked -p omni-omnijev-native OMNIJEV_CHECKPOINT=$PWD/omnijev-v1.1 OMNIJEV_EXPORT=$PWD/omnijev-export \ - cargo test --locked -p omni-omnijev-native -- --include-ignored + cargo test --locked -p omni-omnijev-native --test processing --test heads -- --include-ignored ``` The generator runs the reference's image loading and processor, preparation, layout, diff --git a/recipe/omnijev/export.py b/recipe/omnijev/export.py index 6b6f5474..26cba464 100644 --- a/recipe/omnijev/export.py +++ b/recipe/omnijev/export.py @@ -1,9 +1,11 @@ """Export OmniJev-4B v1.1 for the native worker on CPU. -Merges the language LoRA into the pinned Qwen3.5-4B in BF16, restores the option +By default merges the language LoRA into the pinned Qwen3.5-4B in BF16, restores the option tokens' embeddings, and writes the base vision tensors, the FP32 heads, the tokenizer and the calibration next to it. Use the reference environment in README.md; no GPU -is needed. Needs about 18 GB of host RAM and writes 9 GB. +is needed. --language unmerged-reference preserves the original BF16 base and +separate FP32 adapter for the eager one-shot Score library API. Needs about 18 GB +of host RAM and writes 9 GB. """ import argparse @@ -48,6 +50,8 @@ def main(): parser.add_argument("--base", required=True, type=Path, help=f"{BASE_MODEL_ID} @ {BASE_REVISION}") parser.add_argument("--checkpoint", required=True, type=Path, help=f"{MODEL_ID} @ {CHECKPOINT_REVISION}") parser.add_argument("--out", required=True, type=Path) + parser.add_argument("--language", choices=("merged", "unmerged-reference"), default="merged", + help="unmerged-reference preserves base BF16 weights and the separate FP32 adapter") args = parser.parse_args() if args.out.exists(): raise ValueError("output already exists; choose a new export directory") @@ -94,7 +98,19 @@ def main(): if model.get_output_embeddings().weight.data_ptr() != embedding.data_ptr(): raise ValueError("expected tied input and output embeddings") - model = PeftModel.from_pretrained(model, args.checkpoint).merge_and_unload(safe_merge=True) + unmerged = args.language == "unmerged-reference" + adapter = None + if unmerged: + from safetensors.torch import load_file + adapter = load_file(args.checkpoint / "adapter_model.safetensors") + # This pinned release has 200 projection pairs, all FP32, rank32. + # Do not silently cast an unexpected checkpoint into the reference format. + if len(adapter) != 400 or any(v.dtype != torch.float32 or v.ndim != 2 + or v.shape[0 if ".lora_A." in k else 1] != 32 + for k, v in adapter.items()): + raise ValueError("expected all 400 FP32 rank32 OmniJev adapter tensors") + else: + model = PeftModel.from_pretrained(model, args.checkpoint).merge_and_unload(safe_merge=True) # Write next to the output and rename at the end, so a failed or killed export # leaves no output behind; the next run removes what it left. staging = args.out.with_name(f".{args.out.name}.partial") @@ -102,6 +118,8 @@ def main(): shutil.rmtree(staging) model.model.language_model.save_pretrained(staging, max_shard_size="5GB") model.config.to_json_file(staging / "config.json") + if unmerged: + save_file(adapter, staging / "adapter_model.safetensors") vision = {f"model.visual.{k}": v.contiguous() for k, v in model.model.visual.state_dict().items()} save_file(vision, staging / "vision.safetensors") @@ -118,10 +136,12 @@ def main(): outputs = {p.name: sha256(p) for p in sorted(staging.iterdir()) if p.is_file()} # Written last: the worker refuses incomplete exports. (staging / "omnijev_export.json").write_text(json.dumps({ - "format": "omnijev-merged/1", + "format": "omnijev-unmerged-reference/1" if unmerged else "omnijev-merged/1", "model_id": MODEL_ID, "checkpoint_revision": CHECKPOINT_REVISION, "base_model_id": BASE_MODEL_ID, "base_revision": BASE_REVISION, - "lora": {"rank": 32, "alpha": 64, "merged": "bfloat16"}, + "lora": ({"rank": 32, "alpha": 64, "scale": 2, "dtype": "float32", + "file": "adapter_model.safetensors", "tensors": 400} + if unmerged else {"rank": 32, "alpha": 64, "merged": "bfloat16"}), "vocabulary": VOCABULARY, "option_tokens": OPTION_TOKENS, "max_pixels": MAX_PIXELS, "heads": {"norm": meta["norm"], "lm_features": True, "ordinal": True}, "calibration": {"temperatures": temperatures, "noul_bias": noul_bias}, diff --git a/recipe/omnijev/native.md b/recipe/omnijev/native.md index 02d148b8..77eb9a31 100644 --- a/recipe/omnijev/native.md +++ b/recipe/omnijev/native.md @@ -112,11 +112,16 @@ CUDA library, as absolute paths: ```sh cargo test --locked -p omni-omnijev-native OMNIJEV_EXPORT=$PWD/omnijev-export OMNIJEV_CUDA_LIB=$PWD/target/release/libqwen3_5_cuda.so \ - cargo test --release --locked -p omni-omnijev-native --test worker -- --ignored --test-threads=1 + cargo test --release --locked -p omni-omnijev-native --test worker -- --ignored --test-threads=1 --skip reference QWEN3_5_CHECKPOINT=$PWD/omnijev-export CUA_S1_CUDA_LIB=$PWD/target/release/libqwen3_5_cuda.so \ - cargo test --release --locked -p omni-qwen3-5-native --test readout -- --ignored --test-threads=1 + cargo test --release --locked -p omni-qwen3-5-native --test readout -- --ignored --test-threads=1 --skip reference ``` +These commands select the merged worker's tests. The separate unmerged one-shot +Score library tests require their own artifact and Graph-off settings; see the +[preparation recipe](README.md#separate-unmerged-reference-artifact). They do not +establish a replacement HTTP worker profile. + The direct OmniJev window-Graph regression selects eager and Graph models without changing process-wide environment variables. It compares raw FP32 head bits and answers (excluding timing fields), requires actual capture and warm replay, and diff --git a/src/models/omnijev/README.md b/src/models/omnijev/README.md index 3611a99c..10f6d8bf 100644 --- a/src/models/omnijev/README.md +++ b/src/models/omnijev/README.md @@ -10,6 +10,18 @@ heads, and response finishing, each checked against the pinned reference. The regenerates the fixtures; [the worker recipe](../../../recipe/omnijev/native.md) builds, runs and validates the worker. +An explicit `omnijev-unmerged-reference/1` export and +`Executor::load_one_shot_reference` / `score_one_shot` library API preserve the +separate FP32 LoRA arithmetic for one selected Score question from a complete +prepared request. They reuse the normal full-row Qwen reference execution and +gather the specified final-norm hidden rows, synchronizing both streams before +returning, including errors. The API owns no continuation state. Graphs, reuse, +LM targets and all-question worker execution are unsupported and rejected; +the existing merged worker/defaults remain available. See the +[separate export and validation instructions](../../../recipe/omnijev/README.md#separate-unmerged-reference-artifact). +This bounded library capability does not establish completion of #114's original +all-type 280-answer numerical gate, JPEG fidelity or HTTP acceptance. + ## Pinned artifacts | Artifact | Revision | diff --git a/src/models/omnijev/native/src/executor.rs b/src/models/omnijev/native/src/executor.rs index 2fac1331..bf6a9164 100644 --- a/src/models/omnijev/native/src/executor.rs +++ b/src/models/omnijev/native/src/executor.rs @@ -45,6 +45,7 @@ pub struct Executor { language: Model, heads: Heads, calibration: Calibration, + reference_language: bool, /// Buffers for the state after the request's prefix and after a question's text. Each /// request or question that uses one captures it again; only the allocations outlive a /// request, grown as needed. @@ -113,6 +114,15 @@ struct RowReadout { first: Option, } +/// An explicitly selected Score question from a complete prepared request. +/// This is a one-shot library result, not an all-question serving mode. +pub struct ScoreReadout { + pub zq: Vec, + pub u: Vec>, + pub probabilities: Vec, + pub answer: Value, +} + impl Executor { /// Check the export at `dir`, then load the processor, the vision tower, the /// language model and the heads. @@ -131,9 +141,40 @@ impl Executor { library: &Path, numerics: VisionNumerics, ) -> Result<(Self, Processor)> { - let export = export::load(dir)?; + Self::load_profile(dir, library, numerics, false) + } + + /// Explicit, eager-only unmerged reference library profile. The existing + /// worker/default loader does not select it. Only `score_one_shot` is supported; + /// full requests, prefix reuse, Graphs and LM targets fail closed. + pub fn load_one_shot_reference(dir: &Path, library: &Path) -> Result<(Self, Processor)> { + ensure!( + std::env::var("CUA_S1_GRAPH").as_deref() != Ok("1"), + "one-shot reference requires CUA_S1_GRAPH=0" + ); + Self::load_profile(dir, library, VisionNumerics::Reference, true) + } + + fn load_profile( + dir: &Path, + library: &Path, + numerics: VisionNumerics, + reference_language: bool, + ) -> Result<(Self, Processor)> { + let export = if reference_language { + export::load_reference(dir)? + } else { + export::load(dir)? + }; let processor = Processor::load(dir)?; - let language = Model::load(dir, library)?; + let mut language = if reference_language { + Model::load_with_lora(dir, library, &dir.join(export::ADAPTER), 2.0)? + } else { + Model::load(dir, library)? + }; + if reference_language { + language.enable_reference_text_numerics()?; + } ensure!( language.cfg.image_token_id == Some(IMAGE_TOKEN) && language.cfg.hidden == heads::HIDDEN, @@ -150,6 +191,7 @@ impl Executor { language, heads, calibration: export.calibration, + reference_language, request: None, question: None, }, @@ -197,6 +239,10 @@ impl Executor { prepared: &PreparedRequest, reuse: bool, ) -> Result>> { + ensure!( + !self.reference_language, + "one-shot reference supports only score_one_shot; full requests/reuse/LM targets are unsupported" + ); let result = self.run(prepared, reuse); // Also finish queued work on an error before releasing admission. let vision = self.vision.synchronize(); @@ -207,6 +253,59 @@ impl Executor { Ok(outputs) } + /// Run one Score question, using the original complete request's prefix, + /// positions and readout indices. Other questions must remain in `prepared` + /// so compiling a selection cannot change the longest-common-prefix rule. + pub fn score_one_shot( + &mut self, + prepared: &PreparedRequest, + index: usize, + ) -> Result { + ensure!( + self.reference_language, + "score_one_shot requires the unmerged reference profile" + ); + let question = prepared + .questions + .get(index) + .context("Score question index outside request")?; + ensure!( + question.kind == Kind::Score, + "one-shot reference currently supports Score only; LM targets are unsupported" + ); + let start = Instant::now(); + let result = (|| { + let features = self.vision.forward(&prepared.pixels)?; + let prefix = &prepared.inputs.first().context("no questions")?.token_ids + [..prepared.layout.prefix]; + let input = prepared + .inputs + .get(index) + .context("missing question input")?; + let rows = input + .rows + .iter() + .map(|row| self.row(prefix, row, prepared.grid, &features, false)) + .collect::>>()?; + let zq = rows.first().context("Score has no rows")?.zq.clone(); + let u: Vec<_> = rows.into_iter().map(|row| row.u).collect(); + let probabilities = self.heads.ordinal(&zq, &u)?; + let elapsed = prepared.preparation_seconds + start.elapsed().as_secs_f64(); + let answer = contract::answer(question, &probabilities, &self.calibration, elapsed)?; + Ok(ScoreReadout { + zq, + u, + probabilities, + answer, + }) + })(); + let vision = self.vision.synchronize(); + let language = self.language.synchronize(); + vision?; + language?; + result + } + fn run(&mut self, prepared: &PreparedRequest, reuse: bool) -> Result>> { let features = self.vision.forward(&prepared.pixels)?; let readouts = if reuse { @@ -397,16 +496,20 @@ impl Executor { targets.extend(row.first.map(|token| (at + row.zq, token))); targets.extend(row.targets.iter().map(|&(p, token)| (at + p, token))); } - let readout = self.language.forward_multimodal_readout( - &MultimodalInput { - token_ids: &ids, - image_token_indices: &image_rows, - image_embeddings: features, - position_ids: [&positions[0], &positions[1], &positions[2]], - }, - &[at + row.zq, at + row.u], - &targets, - )?; + let input = MultimodalInput { + token_ids: &ids, + image_token_indices: &image_rows, + image_embeddings: features, + position_ids: [&positions[0], &positions[1], &positions[2]], + }; + let selected = [at + row.zq, at + row.u]; + let readout = if self.reference_language { + self.language + .forward_multimodal_reference_readout(&input, &selected, &targets)? + } else { + self.language + .forward_multimodal_readout(&input, &selected, &targets)? + }; let mut hidden = readout.hidden.into_iter(); let (zq, u) = (hidden.next().unwrap(), hidden.next().unwrap()); let mut logprobs = readout.logprobs.into_iter(); diff --git a/src/models/omnijev/native/src/export.rs b/src/models/omnijev/native/src/export.rs index fb57a361..e7850cfc 100644 --- a/src/models/omnijev/native/src/export.rs +++ b/src/models/omnijev/native/src/export.rs @@ -13,6 +13,8 @@ use crate::contract::{Calibration, MODEL_ID}; use crate::processing::{MAX_PIXELS, OPTION_CLOSE, OPTION_OPEN}; pub const FORMAT: &str = "omnijev-merged/1"; +pub const REFERENCE_FORMAT: &str = "omnijev-unmerged-reference/1"; +pub const ADAPTER: &str = "adapter_model.safetensors"; pub const MANIFEST: &str = "omnijev_export.json"; pub const CHECKPOINT_REVISION: &str = "ffe5f436eaf22e20e2f041f8e74e121fd057a6cb"; pub const BASE_REVISION: &str = "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a"; @@ -36,6 +38,16 @@ pub struct Export { /// Check `dir`'s manifest against the pinned release and every file it lists against /// its SHA-256, then read the calibration. pub fn load(dir: &Path) -> Result { + load_profile(dir, false) +} + +/// Load only an explicitly unmerged artifact. The serving worker's merged loader +/// never accepts this format, and this loader never accepts a merged artifact. +pub fn load_reference(dir: &Path) -> Result { + load_profile(dir, true) +} + +fn load_profile(dir: &Path, reference: bool) -> Result { let path = dir.join(MANIFEST); let manifest: Value = serde_json::from_slice(&fs::read(&path).with_context(|| { @@ -43,7 +55,10 @@ pub fn load(dir: &Path) -> Result { })?) .context("export manifest")?; let expected = [ - ("format", json!(FORMAT)), + ( + "format", + json!(if reference { REFERENCE_FORMAT } else { FORMAT }), + ), ("model_id", json!(MODEL_ID)), ("checkpoint_revision", json!(CHECKPOINT_REVISION)), ("base_revision", json!(BASE_REVISION)), @@ -74,6 +89,24 @@ pub fn load(dir: &Path) -> Result { "export manifest does not list {name}" ); } + let lora = if reference { + json!({"rank": 32, "alpha": 64, "scale": 2, "dtype": "float32", + "file": ADAPTER, "tensors": 400}) + } else { + json!({"rank": 32, "alpha": 64, "merged": "bfloat16"}) + }; + ensure!( + manifest["lora"] == lora, + "export manifest LoRA profile mismatch" + ); + ensure!( + !reference || outputs.contains_key(ADAPTER), + "export manifest does not list {ADAPTER}" + ); + ensure!( + reference || !dir.join(ADAPTER).exists(), + "merged export contains an unmerged adapter" + ); for (name, digest) in outputs { ensure!( !name.is_empty() && !name.contains(['/', '\\']) && name != ".." && name != MANIFEST, @@ -113,11 +146,71 @@ pub fn load(dir: &Path) -> Result { "config.json vocabulary is {}, expected {VOCABULARY}", config["text_config"]["vocab_size"] ); + if reference { + validate_reference_adapter(dir)?; + } Ok(Export { calibration: calibration(&manifest["calibration"])?, }) } +/// The shared LoRA loader accepts partial adapters. OmniJev's reference identity +/// instead requires the complete released module mapping, rank and dtype before +/// any device allocation. The shared loader also checks finite values and offsets. +fn validate_reference_adapter(dir: &Path) -> Result<()> { + use omni_qwen3_5_native::model::Config; + use safetensors::{Dtype, SafeTensors}; + let cfg = Config::load(dir)?; + ensure!( + cfg.hidden == 2560 && cfg.full_attention.len() == 32, + "reference adapter requires OmniJev-4B" + ); + let bytes = fs::read(dir.join(ADAPTER))?; + let tensors = SafeTensors::deserialize(&bytes).context("reference adapter safetensors")?; + ensure!( + tensors.len() == 400, + "reference adapter must contain all 400 tensors" + ); + for (i, &full) in cfg.full_attention.iter().enumerate() { + let mut targets = vec![ + ("mlp.gate_proj", cfg.intermediate, cfg.hidden), + ("mlp.up_proj", cfg.intermediate, cfg.hidden), + ("mlp.down_proj", cfg.hidden, cfg.intermediate), + ]; + if full { + targets.extend([ + ("self_attn.q_proj", 2 * cfg.heads * cfg.head_dim, cfg.hidden), + ("self_attn.k_proj", cfg.kv_heads * cfg.head_dim, cfg.hidden), + ("self_attn.v_proj", cfg.kv_heads * cfg.head_dim, cfg.hidden), + ("self_attn.o_proj", cfg.hidden, cfg.heads * cfg.head_dim), + ]); + } else { + let kd = cfg.lin_k_heads * cfg.lin_k_dim; + let vd = cfg.lin_v_heads * cfg.lin_v_dim; + targets.extend([ + ("linear_attn.in_proj_qkv", 2 * kd + vd, cfg.hidden), + ("linear_attn.in_proj_z", vd, cfg.hidden), + ("linear_attn.out_proj", cfg.hidden, vd), + ]); + } + for (module, rows, cols) in targets { + for (letter, shape) in [("A", [32, cols]), ("B", [rows, 32])] { + let name = format!( + "base_model.model.model.language_model.layers.{i}.{module}.lora_{letter}.weight" + ); + let tensor = tensors + .tensor(&name) + .with_context(|| format!("reference adapter missing {name}"))?; + ensure!( + tensor.dtype() == Dtype::F32 && tensor.shape() == shape, + "reference adapter dtype/shape mismatch for {name}" + ); + } + } + } + Ok(()) +} + /// The manifest's calibration: positive, finite temperatures and a finite Noul bias. pub fn calibration(value: &Value) -> Result { let number = |v: &Value| v.as_f64().filter(|x| x.is_finite()); diff --git a/src/models/qwen3_5/native/src/model.rs b/src/models/qwen3_5/native/src/model.rs index c86e8d9f..993feaa5 100644 --- a/src/models/qwen3_5/native/src/model.rs +++ b/src/models/qwen3_5/native/src/model.rs @@ -1926,6 +1926,55 @@ impl Model { self.run_multimodal(t) } + /// One complete unpadded multimodal row with the unmerged 4B reference + /// arithmetic, gathering final-norm hidden rows in caller order (including + /// duplicates). This uses the normal full-row layer loop and FP32 GDN; + /// it creates no prefix state and never enters the fixed/window path. + /// + /// Select `enable_reference_text_numerics` before calling. Graphs and LM + /// targets are deliberately unsupported by this initial selected-readout + /// contract. Work is synchronized on both success and error before scratch + /// can be reused by another call. + pub fn forward_multimodal_reference_readout( + &mut self, + input: &MultimodalInput<'_>, + rows: &[usize], + targets: &[(usize, u32)], + ) -> Result { + ensure!( + self.cfg.hidden == 2560 + && self.lora.is_some() + && self.reference.is_some() + && self.fp32_gdn.is_some(), + "reference readout requires unmerged 4B reference text numerics" + ); + ensure!(!self.graph_enabled, "reference readout requires Graph off"); + ensure!( + targets.is_empty(), + "reference readout does not support LM targets" + ); + ensure!( + !rows.is_empty() && rows.iter().all(|&r| r < input.token_ids.len()), + "reference readout row outside the prompt or no rows selected" + ); + let result = (|| { + // Reference/LoRA disables the last-row-only MLP optimization, so all + // final-norm rows remain available after this canonical forward. + self.forward_multimodal(input)?; + let s = self.scratch.as_ref().unwrap(); + let hidden = rows + .iter() + .map(|&row| self.hidden_row(s, row)) + .collect::>>()?; + Ok(Readout { + hidden, + logprobs: Vec::new(), + }) + })(); + self.synchronize()?; + result + } + /// Inputs are staged in owned scratch before capture; producer addresses /// never enter the graph, so host and device inputs can share a capture. fn run_multimodal(&mut self, t: usize) -> Result> { diff --git a/tests/omnijev/export.rs b/tests/omnijev/export.rs index 684c5e31..dafd8447 100644 --- a/tests/omnijev/export.rs +++ b/tests/omnijev/export.rs @@ -178,3 +178,34 @@ fn refuses_changed_files_and_other_releases() { dir.error() ); } + +#[test] +fn unmerged_identity_is_explicit_and_profiles_cannot_be_mixed() { + let dir = Dir::new("cross-profile"); + let error = format!("{:#}", export::load_reference(&dir.0).err().unwrap()); + assert!(error.contains("format"), "{error}"); + let mut manifest = dir.manifest(); + manifest["format"] = json!(export::REFERENCE_FORMAT); + dir.write(&manifest); + assert!(dir.error().contains("format")); + assert!( + format!("{:#}", export::load_reference(&dir.0).err().unwrap()).contains("LoRA profile") + ); + manifest["lora"] = json!({"rank": 32, "alpha": 64, "scale": 2, "dtype": "float32", + "file": "adapter_model.safetensors", "tensors": 400}); + dir.write(&manifest); + assert!( + format!("{:#}", export::load_reference(&dir.0).err().unwrap()) + .contains("does not list adapter_model") + ); + // The complete adapter must be hashed, never inherited implicitly from a + // sibling export. A merged directory must reject a stray adapter as well. + let dir = Dir::new("stray-adapter"); + fs::write(dir.0.join(export::ADAPTER), "unmerged").unwrap(); + assert!(dir.error().contains("unmerged adapter")); + let dir = Dir::new("lora-scale"); + let mut manifest = dir.manifest(); + manifest["lora"]["alpha"] = json!(32); + dir.write(&manifest); + assert!(dir.error().contains("LoRA profile")); +} diff --git a/tests/omnijev/window_graph.rs b/tests/omnijev/window_graph.rs index d3ce8f1f..234450b6 100644 --- a/tests/omnijev/window_graph.rs +++ b/tests/omnijev/window_graph.rs @@ -27,6 +27,7 @@ fn language_mode(executor: Executor, dir: &Path, library: &Path, graph: bool) -> language, heads, calibration, + reference_language, request, question, } = executor; @@ -36,6 +37,7 @@ fn language_mode(executor: Executor, dir: &Path, library: &Path, graph: bool) -> language: Model::load_with_graph(dir, library, graph).unwrap(), heads, calibration, + reference_language, request: None, question: None, } diff --git a/tests/omnijev/worker.rs b/tests/omnijev/worker.rs index 9cab96a9..fa6eb0e5 100644 --- a/tests/omnijev/worker.rs +++ b/tests/omnijev/worker.rs @@ -3,7 +3,8 @@ //! libqwen3_5_cuda.so, both absolute: //! //! OMNIJEV_EXPORT= OMNIJEV_CUDA_LIB=$PWD/target/release/libqwen3_5_cuda.so \ -//! cargo test --release -p omni-omnijev-native --test worker -- --ignored --test-threads=1 +//! cargo test --release -p omni-omnijev-native --test worker -- --ignored --test-threads=1 --skip reference +//! The separate reference tests below name their additional prerequisites. mod common; use std::path::PathBuf; @@ -11,6 +12,97 @@ use std::path::PathBuf; use omni_omnijev_native::executor::{self, Executor}; use serde_json::{Value, json}; +/// Real original #114 regression: compilation always sees all twelve questions. +/// The externally supplied request/oracle stay private and hash-pinned by the +/// validation recipe; no source images or checkpoint tensors are checked in. +#[test] +#[ignore = "requires CUDA, OMNIJEV_REFERENCE_EXPORT, OMNIJEV_CUDA_LIB, OMNIJEV_ORIGINAL_REQUEST and OMNIJEV_REFERENCE_ORACLE; CUA_S1_GRAPH=0"] +fn unmerged_reference_original_q2_gate() { + use sha2::{Digest, Sha256}; + let env_path = |key| std::path::PathBuf::from(std::env::var_os(key).expect(key)); + let dir = env_path("OMNIJEV_REFERENCE_EXPORT"); + let (mut executor, processor) = + executor::Executor::load_one_shot_reference(&dir, &env_path("OMNIJEV_CUDA_LIB")).unwrap(); + let raw = std::fs::read(env_path("OMNIJEV_ORIGINAL_REQUEST")).unwrap(); + assert_eq!( + format!("{:x}", Sha256::digest(&raw)), + "a34c2d136a27d11a4c9085547feda474cb1f6e5d44fefcf9cfcd410c7ee5fab5" + ); + let prepared = processor.prepare(&raw).unwrap(); + assert_eq!(prepared.questions.len(), 12); + assert_eq!(prepared.layout.prefix, 581); + assert_eq!(prepared.layout.processed_tokens, 1214); + let index = prepared + .questions + .iter() + .position(|q| q.id == "q2") + .unwrap(); + assert!(executor.execute(&prepared).is_err()); + assert!(executor.execute_rows(&prepared).is_err()); + assert!( + executor + .score_one_shot(&prepared, prepared.questions.len()) + .is_err() + ); + let nonscore = prepared + .questions + .iter() + .position(|q| q.kind != omni_omnijev_native::contract::Kind::Score) + .unwrap(); + assert!(executor.score_one_shot(&prepared, nonscore).is_err()); + let read = executor.score_one_shot(&prepared, index).unwrap(); + let repeat = executor.score_one_shot(&prepared, index).unwrap(); + assert_eq!(read.zq, repeat.zq); + assert_eq!(read.u, repeat.u); + assert_eq!(read.probabilities, repeat.probabilities); + if let Some(path) = std::env::var_os("OMNIJEV_REFERENCE_RESULT") { + std::fs::write( + path, + serde_json::to_vec_pretty(&json!({ + "question_count": prepared.questions.len(), "prefix": prepared.layout.prefix, + "input_tokens": prepared.layout.processed_tokens, "qid": "q2", + "zq": read.zq, "u": read.u, "mu": read.probabilities, "answer": read.answer, + "repeat_exact": true, "unsupported_modes_rejected": true, + })) + .unwrap(), + ) + .unwrap(); + } + let oracle: Value = + serde_json::from_slice(&std::fs::read(env_path("OMNIJEV_REFERENCE_ORACLE")).unwrap()) + .unwrap(); + let expected = &oracle["official_full_answer"]["q2"]; + assert_eq!(read.answer["score"], expected["score"]); + let delta = (read.answer["confidence"].as_f64().unwrap() + - expected["confidence"].as_f64().unwrap()) + .abs(); + assert!( + delta <= 0.01, + "original q2 confidence delta {delta}: native={}, reference={expected}", + read.answer + ); + for (key, p) in read.answer["probabilities"].as_object().unwrap() { + let e = &expected["probabilities"][key]; + assert!( + (p.as_f64().unwrap() - e.as_f64().unwrap()).abs() <= 0.01, + "level {key}" + ); + } +} + +#[test] +#[ignore = "run with CUA_S1_GRAPH=1; verifies rejection before reading artifacts or loading CUDA"] +fn reference_graph_is_rejected_before_loading() { + assert_eq!(std::env::var("CUA_S1_GRAPH").as_deref(), Ok("1")); + let error = Executor::load_one_shot_reference( + std::path::Path::new("/missing-export"), + std::path::Path::new("/missing-cuda-library"), + ) + .err() + .unwrap(); + assert!(error.to_string().contains("CUA_S1_GRAPH=0")); +} + fn executor() -> (Executor, omni_omnijev_native::processing::Processor) { let var = |name: &str| PathBuf::from(std::env::var_os(name).unwrap_or_else(|| panic!("set {name}"))); diff --git a/tests/qwen3_5/readout.rs b/tests/qwen3_5/readout.rs index d0ccdf37..55ab811f 100644 --- a/tests/qwen3_5/readout.rs +++ b/tests/qwen3_5/readout.rs @@ -3,7 +3,7 @@ //! multimodal Qwen3.5 export with `image_token_id`, such as an OmniJev export: //! //! CUA_S1_CUDA_LIB=$PWD/target/release/libqwen3_5_cuda.so QWEN3_5_CHECKPOINT= \ -//! cargo test --release -p omni-qwen3-5-native --test readout -- --ignored --test-threads=1 +//! cargo test --release -p omni-qwen3-5-native --test readout -- --ignored --test-threads=1 --skip reference use std::fs::File; use std::path::PathBuf; @@ -258,3 +258,56 @@ fn windows_with_captures_equal_one_pass() { .is_err() ); } + +#[test] +#[ignore = "requires CUDA, an unmerged OmniJev export and QWEN3_5_ADAPTER; run serially with CUA_S1_GRAPH=0"] +fn reference_selected_rows_preserve_full_row_and_recover() { + let library = PathBuf::from(std::env::var_os("CUA_S1_CUDA_LIB").unwrap()); + let adapter = PathBuf::from(std::env::var_os("QWEN3_5_ADAPTER").unwrap()); + let mut model = Model::load_with_lora(&checkpoint(), &library, &adapter, 2.).unwrap(); + model.enable_reference_text_numerics().unwrap(); + let original = Prompt::new(&model, 50, 75); + let t = original.ids.len(); + let last = model.forward_multimodal(&original.input()).unwrap(); + let read = model + .forward_multimodal_reference_readout(&original.input(), &[t - 1, 10, t - 1, 0], &[]) + .unwrap(); + assert_eq!(read.hidden[0], last); + assert_eq!(read.hidden[0], read.hidden[2]); + assert!(read.logprobs.is_empty()); + assert!( + model + .forward_multimodal_reference_readout(&original.input(), &[t], &[]) + .is_err() + ); + assert!( + model + .forward_multimodal_reference_readout(&original.input(), &[], &[]) + .is_err() + ); + assert!( + model + .forward_multimodal_reference_readout(&original.input(), &[0], &[(0, 1000)]) + .is_err() + ); + // Existing fixed/window and prefix guards must still reject this arithmetic. + assert!( + model + .forward_multimodal_readout(&original.input(), &[0], &[]) + .is_err() + ); + assert!(model.alloc_prefix(64).is_err()); + let grown = Prompt::new(&model, 50, 300); + model + .forward_multimodal_reference_readout(&grown.input(), &[0, grown.ids.len() - 1], &[]) + .unwrap(); + let again = model + .forward_multimodal_reference_readout(&original.input(), &[t - 1, 10, t - 1, 0], &[]) + .unwrap(); + assert_eq!( + bits(&again.hidden), + bits(&read.hidden), + "request state/scratch growth leaked into selected rows" + ); + assert_eq!(model.graph_stats().captures, 0); +} From 46c1ae5ca784dd0c0e3c2c00fbc0691d018521ad Mon Sep 17 00:00:00 2001 From: psv666 <2693925048@qq.com> Date: Fri, 20 Nov 2026 14:13:36 +0800 Subject: [PATCH 3/4] omnijev: execute explicit unmerged LM projection groups --- src/models/omnijev/native/src/executor.rs | 140 ++++++++++++- src/models/omnijev/native/src/main.rs | 10 +- src/models/omnijev/native/src/processing.rs | 86 +++++++- src/models/qwen3_5/native/src/model.rs | 172 +++++++++++++++ tests/omnijev/processing.rs | 68 ++++++ tests/omnijev/worker.rs | 221 +++++++++++++++++++- tests/qwen3_5/readout.rs | 95 +++++++++ 7 files changed, 779 insertions(+), 13 deletions(-) diff --git a/src/models/omnijev/native/src/executor.rs b/src/models/omnijev/native/src/executor.rs index bf6a9164..bf45c5cf 100644 --- a/src/models/omnijev/native/src/executor.rs +++ b/src/models/omnijev/native/src/executor.rs @@ -10,14 +10,14 @@ use std::time::Instant; use anyhow::{Context, Result, ensure}; use omni_qwen3_5_native::inputs::{MultimodalInput, image_positions}; -use omni_qwen3_5_native::model::{Model, PrefixState}; +use omni_qwen3_5_native::model::{Model, PrefixState, ReferenceLmTarget}; use omni_qwen3_5_native::vision::VisionModel; use serde_json::{Map, Value, json}; use crate::contract::{self, Calibration, Kind, round4}; use crate::export; use crate::heads::{self, Heads}; -use crate::processing::{IMAGE_TOKEN, PreparedRequest, Processor, Row}; +use crate::processing::{self, IMAGE_TOKEN, PreparedRequest, Processor, Row}; #[cfg(test)] #[path = "../../../../../tests/omnijev/window_graph.rs"] @@ -145,8 +145,8 @@ impl Executor { } /// Explicit, eager-only unmerged reference library profile. The existing - /// worker/default loader does not select it. Only `score_one_shot` is supported; - /// full requests, prefix reuse, Graphs and LM targets fail closed. + /// worker/default loader does not select it. Complete requests execute normal + /// independent rows and explicit request-chunk LM groups; no reuse or Graphs. pub fn load_one_shot_reference(dir: &Path, library: &Path) -> Result<(Self, Processor)> { ensure!( std::env::var("CUA_S1_GRAPH").as_deref() != Ok("1"), @@ -166,7 +166,11 @@ impl Executor { } else { export::load(dir)? }; - let processor = Processor::load(dir)?; + let processor = if reference_language { + Processor::load_one_shot_reference(dir)? + } else { + Processor::load(dir)? + }; let mut language = if reference_language { Model::load_with_lora(dir, library, &dir.join(export::ADAPTER), 2.0)? } else { @@ -203,7 +207,7 @@ impl Executor { /// `latency_total_s`: the request's preparation, its vision and language passes and /// its heads, over its questions and in all. pub fn execute(&mut self, prepared: &PreparedRequest) -> Result> { - self.execute_with(prepared, true) + self.execute_with(prepared, !self.reference_language) } /// `execute` with one plain pass per (question, option) row over the prefix and that @@ -240,10 +244,14 @@ impl Executor { reuse: bool, ) -> Result>> { ensure!( - !self.reference_language, - "one-shot reference supports only score_one_shot; full requests/reuse/LM targets are unsupported" + !self.reference_language || !reuse, + "one-shot reference does not support prefix reuse" ); - let result = self.run(prepared, reuse); + let result = if self.reference_language { + self.run_reference(prepared) + } else { + self.run(prepared, reuse) + }; // Also finish queued work on an error before releasing admission. let vision = self.vision.synchronize(); let language = self.language.synchronize(); @@ -253,6 +261,120 @@ impl Executor { Ok(outputs) } + fn run_reference(&mut self, prepared: &PreparedRequest) -> Result>> { + ensure!( + !prepared.questions.is_empty() + && prepared.questions.len() == prepared.inputs.len() + && prepared + .questions + .iter() + .zip(&prepared.inputs) + .all(|(q, input)| !input.rows.is_empty() && q.keys.len() == input.rows.len()), + "reference question/row cardinality mismatch" + ); + let rows: Vec<&Row> = prepared + .inputs + .iter() + .flat_map(|input| &input.rows) + .collect(); + let chunks = processing::reference_lm_chunks(&rows)?; + let prefix = prepared.inputs[0] + .token_ids + .get(..prepared.layout.prefix) + .context("reference prefix outside prompt")?; + let features = self.vision.forward(&prepared.pixels)?; + let mut readouts = Vec::with_capacity(rows.len()); + for chunk in chunks { + let sub = &rows[chunk.rows]; + let mut hidden = Vec::with_capacity(sub.len()); + for row in sub { + let ids: Vec = prefix.iter().chain(&row.tokens).copied().collect(); + let positions = image_positions(&ids, IMAGE_TOKEN, prepared.grid)?; + let images: Vec = prefix + .iter() + .enumerate() + .filter_map(|(i, &id)| (id == IMAGE_TOKEN).then_some(i)) + .collect(); + let input = MultimodalInput { + token_ids: &ids, + image_token_indices: &images, + image_embeddings: &features, + position_ids: [&positions[0], &positions[1], &positions[2]], + }; + let at = prefix.len(); + let selected: Vec = [at + row.zq, at + row.u] + .into_iter() + .chain(row.targets.iter().map(|&(p, _)| at + p)) + .collect(); + hidden.push( + self.language + .forward_multimodal_reference_hidden(&input, &selected)?, + ); + } + let options: Vec<_> = chunk + .option_targets + .iter() + .map(|&(i, row, token)| ReferenceLmTarget { + hidden: &hidden[i], + row, + token, + }) + .collect(); + let picked = self.language.reference_lm_logprobs(&options)?; + let firsts: Vec<_> = chunk + .first_targets + .iter() + .map(|&(i, token)| ReferenceLmTarget { + hidden: &hidden[i], + row: 0, + token, + }) + .collect(); + let first = self.language.reference_lm_logprobs(&firsts)?; + let mut chunk_readouts: Vec<_> = hidden + .iter() + .map(|h| { + Ok(RowReadout { + zq: h.row(0)?.iter().map(|v| v.to_f32()).collect(), + u: h.row(1)?.iter().map(|v| v.to_f32()).collect(), + picked: Vec::new(), + first: None, + }) + }) + .collect::>()?; + for ((i, _, _), p) in chunk.option_targets.iter().zip(picked) { + chunk_readouts[*i].picked.push(p); + } + for ((i, _), p) in chunk.first_targets.iter().zip(first) { + chunk_readouts[*i].first = Some(p); + } + readouts.extend(chunk_readouts); + } + let mut at = 0; + prepared + .questions + .iter() + .zip(&prepared.inputs) + .map(|(q, input)| { + let group = &readouts[at..at + input.rows.len()]; + at += input.rows.len(); + let u: Vec<_> = group.iter().map(|r| r.u.clone()).collect(); + if q.kind == Kind::Score { + self.heads.ordinal(&group[0].zq, &u) + } else { + let picked: Vec<_> = group.iter().map(|r| r.picked.clone()).collect(); + let first: Vec<_> = group.iter().map(|r| r.first).collect(); + self.heads.option_probabilities( + &u, + &group[0].zq, + q.kind.type_id(), + &heads::lm_features(&picked, &first), + ) + } + }) + .collect() + } + /// Run one Score question, using the original complete request's prefix, /// positions and readout indices. Other questions must remain in `prepared` /// so compiling a selection cannot change the longest-common-prefix rule. diff --git a/src/models/omnijev/native/src/main.rs b/src/models/omnijev/native/src/main.rs index 6cf4882d..37080e87 100644 --- a/src/models/omnijev/native/src/main.rs +++ b/src/models/omnijev/native/src/main.rs @@ -160,7 +160,15 @@ async fn main() -> Result<()> { .map(PathBuf::from) .map_or_else(cuda::default_library, Ok)?; let shared = tokio::task::spawn_blocking(move || -> Result { - let (mut executor, processor) = Executor::load(&dir, &library)?; + let profile = std::env::var("OMNIJEV_PROFILE").or_else(|error| match error { + std::env::VarError::NotPresent => Ok("merged-fixed".to_owned()), + error => Err(error), + })?; + let (mut executor, processor) = match profile.as_str() { + "merged-fixed" => Executor::load(&dir, &library)?, + "unmerged-reference" => Executor::load_one_shot_reference(&dir, &library)?, + _ => anyhow::bail!("OMNIJEV_PROFILE must be merged-fixed or unmerged-reference"), + }; let warmup = processor.prepare(&warmup_request())?; executor.execute(&warmup).context("warmup request")?; Ok(Shared { diff --git a/src/models/omnijev/native/src/processing.rs b/src/models/omnijev/native/src/processing.rs index 9af33bce..6dfc9333 100644 --- a/src/models/omnijev/native/src/processing.rs +++ b/src/models/omnijev/native/src/processing.rs @@ -106,6 +106,74 @@ pub struct Row { pub first: Option, } +/// One official request-row chunk (default MSO_BRANCH_CHUNK=64). Targets name a +/// chunk-local row, its owned selected-hidden index, and the vocabulary token. +/// Selected-hidden order is zq, u, then every option prediction in token order. +#[derive(Debug, PartialEq)] +pub struct ReferenceLmChunk { + pub rows: std::ops::Range, + pub option_targets: Vec<(usize, usize, u32)>, + pub first_targets: Vec<(usize, u32)>, +} + +/// Preserve complete-request row/token order, including Score's unused LM rows. +/// Group normalization remains per question after all chunks have executed. +pub fn reference_lm_chunks(rows: &[&Row]) -> Result> { + ensure!(!rows.is_empty(), "reference request has no rows"); + let mut count = 0usize; + for row in rows { + ensure!( + row.zq < row.tokens.len() + && row.targets.len() <= MAX_TARGETS + && row.zq.checked_add(2 + row.targets.len()) == Some(row.u) + && row.u.checked_add(1) == Some(row.tokens.len()) + && row.tokens.get(row.zq + 1) == Some(&OPTION_OPEN) + && row.tokens.get(row.u) == Some(&OPTION_CLOSE) + && row.first == row.targets.first().map(|t| t.1) + && row + .targets + .iter() + .enumerate() + .all(|(i, &(p, token))| p == row.zq + 1 + i + && row.tokens.get(p + 1) == Some(&token) + && token < 248_079), + "invalid reference option row/target layout" + ); + count = count + .checked_add(row.targets.len() + usize::from(row.first.is_some())) + .context("reference target count overflow")?; + } + ensure!( + count <= MAX_TARGETS, + "reference request exceeds LM target budget" + ); + Ok(rows + .chunks(64) + .enumerate() + .map(|(chunk, sub)| { + let start = chunk * 64; + ReferenceLmChunk { + rows: start..start + sub.len(), + option_targets: sub + .iter() + .enumerate() + .flat_map(|(i, row)| { + row.targets + .iter() + .enumerate() + .map(move |(j, &(_, token))| (i, j + 2, token)) + }) + .collect(), + first_targets: sub + .iter() + .enumerate() + .filter_map(|(i, row)| row.first.map(|t| (i, t))) + .collect(), + } + }) + .collect()) +} + pub struct PreparedQuestion { /// The single-question sequence: prompt, image and every option block. pub token_ids: Vec, @@ -196,6 +264,7 @@ pub fn layout(sequences: &[Vec], grid: [usize; 3]) -> Result<(Layout, Vec Result { + Ok(Self { + reference_lm: true, + ..Self::load(dir)? + }) } /// The single-question sequence: the prompt's text tokens around the image's @@ -283,6 +364,9 @@ impl Processor { targets <= MAX_TARGETS, "request reads {targets} option-text tokens, above {MAX_TARGETS}" ); + if self.reference_lm { + reference_lm_chunks(&rows.iter().flatten().collect::>())?; + } let inputs = sequences .into_iter() .zip(rows) diff --git a/src/models/qwen3_5/native/src/model.rs b/src/models/qwen3_5/native/src/model.rs index 993feaa5..04e643b6 100644 --- a/src/models/qwen3_5/native/src/model.rs +++ b/src/models/qwen3_5/native/src/model.rs @@ -782,6 +782,56 @@ pub struct Readout { pub logprobs: Vec, } +/// Owned CPU BF16 final-norm rows. These bytes survive subsequent forwards and +/// scratch growth; they contain no device pointers or prefix state. +#[derive(Debug)] +pub struct Bf16HiddenRows { + width: usize, + values: Vec, +} + +impl Bf16HiddenRows { + pub fn new(width: usize, values: Vec) -> Result { + ensure!( + width > 0 && values.len().is_multiple_of(width), + "hidden matrix has an invalid width or row extent" + ); + ensure!( + values.iter().all(|v| v.is_finite()), + "non-finite hidden matrix" + ); + Ok(Self { width, values }) + } + + pub fn rows(&self) -> usize { + self.values.len() / self.width + } + + pub fn row(&self, index: usize) -> Result<&[half::bf16]> { + ensure!(index < self.rows(), "hidden row outside owned matrix"); + Ok(&self.values[index * self.width..(index + 1) * self.width]) + } +} + +/// One row of a caller-declared LM projection group. The entire ordered slice is +/// one GEMM: duplicate rows/targets are retained and it is never split at 64. +pub struct ReferenceLmTarget<'a> { + pub hidden: &'a Bf16HiddenRows, + pub row: usize, + pub token: u32, +} + +/// Bound one explicitly declared projection's device/host logits allocation. +/// OmniJev's processor also bounds the complete request to this target count. +pub const MAX_REFERENCE_LM_ROWS: usize = 16_384; + +struct ReferenceVocabulary { + capacity: usize, + hidden: DeviceBuffer, + logits: DeviceBuffer, + host: Vec, +} + /// log-softmax of one row of bfloat16 logits over its first `vocab` entries, at `token`: /// the logits as float32, their maximum, and a float64 sum of the exponentials. pub(crate) fn token_logprob(logits: &[u8], vocab: usize, token: u32) -> f32 { @@ -881,6 +931,7 @@ pub struct Model { /// Hidden rows and logits for `forward_multimodal_readout`'s vocabulary pass, and /// the host copy of the logits; allocated on first use. vocab_buffers: Option<(DeviceBuffer, DeviceBuffer, Vec)>, + reference_vocab: Option, /// Reusable row-index storage; contains no request values. compaction_maps: Option<(usize, DeviceBuffer)>, /// One layer's FP32 recurrent state and prefix V, overwritten on every use. @@ -1060,6 +1111,7 @@ impl Model { scratch: None, shared: [None, None], vocab_buffers: None, + reference_vocab: None, compaction_maps: None, tree_workspace: None, graph_enabled: graph, @@ -1975,6 +2027,126 @@ impl Model { result } + /// Normal full-row reference execution, copying only the declared final-norm + /// rows into an owned BF16 matrix. The existing selected-readout guards still + /// apply; LM projection is a separate operation after all chunk rows exist. + pub fn forward_multimodal_reference_hidden( + &mut self, + input: &MultimodalInput<'_>, + rows: &[usize], + ) -> Result { + let read = self.forward_multimodal_reference_readout(input, rows, &[])?; + // BF16 -> F32 -> BF16 is value exact. The synchronized readout owns every + // value before this call returns, independently of normal forward scratch. + Bf16HiddenRows::new( + self.cfg.hidden, + read.hidden + .into_iter() + .flatten() + .map(half::bf16::from_f32) + .collect(), + ) + } + + /// One complete, caller-ordered full-vocabulary projection using reference + /// GEMM arithmetic and contiguous BF16 logits (stride equals actual vocab). + /// Hidden matrices are owned by the caller; no current-forward scratch is read. + /// Empty groups perform no device work. The existing CPU FP64 log-softmax is + /// reused and is approximate to the framework's FP32 reduction, not bit exact. + pub fn reference_lm_logprobs(&mut self, targets: &[ReferenceLmTarget<'_>]) -> Result> { + ensure!( + self.cfg.hidden == 2560 + && self.lora.is_some() + && self.reference.is_some() + && self.fp32_gdn.is_some() + && !self.graph_enabled, + "LM projection requires eager unmerged 4B reference text numerics" + ); + ensure!( + targets.len() <= MAX_REFERENCE_LM_ROWS, + "LM projection group exceeds row budget" + ); + let vocab = self.embed.shape[0]; + ensure!( + targets.iter().all(|t| t.hidden.width == self.cfg.hidden + && t.row < t.hidden.rows() + && (t.token as usize) < vocab), + "LM projection hidden width, row or token outside matrix/vocabulary" + ); + if targets.is_empty() { + return Ok(Vec::new()); + } + let m = targets.len(); + let hb = self.cfg.hidden * BF16; + let lb = vocab + .checked_mul(BF16) + .context("LM vocabulary byte extent overflow")?; + let hidden_bytes = m + .checked_mul(hb) + .context("LM hidden byte extent overflow")?; + let logits_bytes = m + .checked_mul(lb) + .context("LM logits byte extent overflow")?; + let mut input = Vec::with_capacity(hidden_bytes); + for target in targets { + for value in target.hidden.row(target.row)? { + input.extend_from_slice(&value.to_le_bytes()); + } + } + let result = (|| { + cuda::set_device(0)?; + if self.reference_vocab.as_ref().is_none_or(|v| v.capacity < m) { + // Finish old queued consumers before replacing allocations, even + // if a later allocation fails. Only a complete new set is stored. + self.synchronize()?; + self.reference_vocab = None; + self.reference_vocab = Some(ReferenceVocabulary { + capacity: m, + hidden: DeviceBuffer::new(hidden_bytes)?, + logits: DeviceBuffer::new(logits_bytes)?, + host: vec![0; logits_bytes], + }); + } + let buffers = self.reference_vocab.as_mut().unwrap(); + // SAFETY: all inputs and outputs have checked m*width extents; the + // tied embedding is [vocab,hidden]. No padding enters the denominator. + unsafe { + cuda::upload(buffers.hidden.at(0), &input, self.stream)?; + check( + (cuda::api().cs1_gemm)( + self.gemm, + buffers.hidden.at(0), + self.embed.ptr, + buffers.logits.at(0), + m as i32, + vocab as i32, + self.cfg.hidden as i32, + vocab as i32, + self.stream, + ), + "reference vocabulary gemm", + )?; + cuda::download( + &mut buffers.host[..logits_bytes], + buffers.logits.at(0), + self.stream, + )?; + } + let logprobs: Vec = targets + .iter() + .enumerate() + .map(|(i, t)| token_logprob(&buffers.host[i * lb..(i + 1) * lb], vocab, t.token)) + .collect(); + ensure!( + logprobs.iter().all(|p| p.is_finite() && *p <= 0.0), + "invalid reference LM log-probability" + ); + Ok(logprobs) + })(); + self.synchronize()?; + result + } + /// Inputs are staged in owned scratch before capture; producer addresses /// never enter the graph, so host and device inputs can share a capture. fn run_multimodal(&mut self, t: usize) -> Result> { diff --git a/tests/omnijev/processing.rs b/tests/omnijev/processing.rs index f1ad09dd..a688ab3a 100644 --- a/tests/omnijev/processing.rs +++ b/tests/omnijev/processing.rs @@ -6,6 +6,74 @@ use omni_omnijev_native::processing::{self, Processor}; use omni_qwen3_5_native::inputs::image_positions; use sha2::{Digest, Sha256}; +#[test] +fn reference_lm_chunks_keep_request_rows_and_inner_token_order() { + use processing::{OPTION_CLOSE, OPTION_OPEN, Row, reference_lm_chunks}; + let mut rows: Vec = (0..65) + .map(|i| Row { + tokens: vec![10, OPTION_OPEN, 100 + i, 200 + i, OPTION_CLOSE], + zq: 0, + u: 4, + targets: vec![(1, 100 + i), (2, 200 + i)], + first: Some(100 + i), + }) + .collect(); + // An empty option stays in the chunk's row extent without projection targets. + rows[7] = Row { + tokens: vec![10, OPTION_OPEN, OPTION_CLOSE], + zq: 0, + u: 2, + targets: vec![], + first: None, + }; + let chunks = reference_lm_chunks(&rows.iter().collect::>()).unwrap(); + assert_eq!(chunks.len(), 2); + assert_eq!(chunks[0].rows, 0..64); + assert_eq!(chunks[1].rows, 64..65); + assert_eq!(chunks[0].option_targets.len(), 126); + assert_eq!(chunks[0].first_targets.len(), 63); + assert_eq!(chunks[0].option_targets[0..2], [(0, 2, 100), (0, 3, 200)]); + assert_eq!(chunks[0].option_targets[14..16], [(8, 2, 108), (8, 3, 208)]); + assert_eq!(chunks[1].option_targets, [(0, 2, 164), (0, 3, 264)]); + assert_eq!(chunks[1].first_targets, [(0, 164)]); + assert!(reference_lm_chunks(&[]).is_err()); + rows[0].first = None; + assert!(reference_lm_chunks(&rows.iter().collect::>()).is_err()); + rows[0].first = Some(100); + rows[0].targets.swap(0, 1); + assert!(reference_lm_chunks(&rows.iter().collect::>()).is_err()); + rows[0].zq = usize::MAX; + assert!(reference_lm_chunks(&rows.iter().collect::>()).is_err()); +} + +#[test] +fn reference_lm_chunk_budget_and_empty_option_policy() { + let empty = processing::Row { + tokens: vec![10, processing::OPTION_OPEN, processing::OPTION_CLOSE], + zq: 0, + u: 2, + targets: vec![], + first: None, + }; + let chunks = processing::reference_lm_chunks(&[&empty]).unwrap(); + assert!(chunks[0].option_targets.is_empty() && chunks[0].first_targets.is_empty()); + let mut full = processing::Row { + tokens: vec![10, processing::OPTION_OPEN, 100, processing::OPTION_CLOSE], + zq: 0, + u: 3, + targets: vec![(1, 100)], + first: Some(100), + }; + let rows = vec![&full; processing::MAX_TARGETS / 2]; + assert!(processing::reference_lm_chunks(&rows).is_ok()); + let rows = vec![&full; processing::MAX_TARGETS / 2 + 1]; + assert!(processing::reference_lm_chunks(&rows).is_err()); + full.tokens[2] = 248_079; + full.targets[0].1 = 248_079; + full.first = Some(248_079); + assert!(processing::reference_lm_chunks(&[&full]).is_err()); +} + fn positions_sha256(positions: &[Vec; 3]) -> String { let mut hash = Sha256::new(); for axis in positions { diff --git a/tests/omnijev/worker.rs b/tests/omnijev/worker.rs index fa6eb0e5..8e8d892c 100644 --- a/tests/omnijev/worker.rs +++ b/tests/omnijev/worker.rs @@ -12,6 +12,224 @@ use std::path::PathBuf; use omni_omnijev_native::executor::{self, Executor}; use serde_json::{Value, json}; +/// Actual hash-pinned complete-request tensors qualify the Rust projection, +/// reduction, six features and heads without backbone/topology differences. +#[test] +#[ignore = "requires original OMNIJEV_LM_CAPTURE/OMNIJEV_LM_LAYOUT and unmerged reference model/CUDA; CUA_S1_GRAPH=0"] +fn reference_actual_lm_groups_and_features() { + use half::bf16; + use omni_omnijev_native::{ + contract::Kind, + heads::{self, Heads}, + }; + use omni_qwen3_5_native::model::{Bf16HiddenRows, Model, ReferenceLmTarget}; + use safetensors::{Dtype, SafeTensors}; + use sha2::{Digest, Sha256}; + let path = |key| PathBuf::from(std::env::var_os(key).expect(key)); + let bytes = std::fs::read(path("OMNIJEV_LM_CAPTURE")).unwrap(); + assert_eq!( + format!("{:x}", Sha256::digest(&bytes)), + "fb7032237649393af91922c29ff6a3317c4bcd3eb64fc683210d9f9dc03aced7" + ); + let tensors = SafeTensors::deserialize(&bytes).unwrap(); + let layout: Value = + serde_json::from_slice(&std::fs::read(path("OMNIJEV_LM_LAYOUT")).unwrap()).unwrap(); + assert_eq!( + layout["request_sha256"], + "a34c2d136a27d11a4c9085547feda474cb1f6e5d44fefcf9cfcd410c7ee5fab5" + ); + let float = |name: &str| { + let tensor = tensors.tensor(name).unwrap(); + match tensor.dtype() { + Dtype::F32 => tensor + .data() + .chunks_exact(4) + .map(|b| f32::from_le_bytes(b.try_into().unwrap())) + .collect::>(), + Dtype::BF16 => tensor + .data() + .chunks_exact(2) + .map(|b| bf16::from_le_bytes(b.try_into().unwrap()).to_f32()) + .collect(), + other => panic!("unexpected tensor dtype {other:?}"), + } + }; + let dir = path("OMNIJEV_REFERENCE_EXPORT"); + let mut model = Model::load_with_lora( + &dir, + &path("OMNIJEV_CUDA_LIB"), + &dir.join("adapter_model.safetensors"), + 2., + ) + .unwrap(); + model.enable_reference_text_numerics().unwrap(); + let mut picked = vec![Vec::new(); 35]; + let mut first = vec![None; 35]; + let mut metrics = serde_json::Map::new(); + for (name, order_name, m) in [("option", "option_order", 112), ("first", "zq_order", 31)] { + let tensor = tensors.tensor(&format!("{name}.hidden")).unwrap(); + assert_eq!(tensor.shape(), &[m, 2560]); + assert_eq!(tensor.dtype(), Dtype::BF16); + let hidden = Bf16HiddenRows::new( + 2560, + tensor + .data() + .chunks_exact(2) + .map(|b| bf16::from_le_bytes(b.try_into().unwrap())) + .collect(), + ) + .unwrap(); + let order = layout["chunks"][0][order_name].as_array().unwrap(); + let targets: Vec<_> = order + .iter() + .enumerate() + .map(|(row, target)| ReferenceLmTarget { + hidden: &hidden, + row, + token: target["target"].as_u64().unwrap() as u32, + }) + .collect(); + let actual = model.reference_lm_logprobs(&targets).unwrap(); + let expected = float(&format!("{name}.picked")); + assert_eq!(actual.len(), expected.len()); + let max = actual + .iter() + .zip(&expected) + .map(|(a, e)| (a - e).abs()) + .fold(0.0f32, f32::max); + assert!(max <= 1e-5, "{name} picked log-prob maxabs {max}"); + let different = actual + .iter() + .zip(&expected) + .filter(|(a, e)| a.to_bits() != e.to_bits()) + .count(); + let l2 = (actual + .iter() + .zip(&expected) + .map(|(a, e)| (*a as f64 - *e as f64).powi(2)) + .sum::() + / expected.iter().map(|e| (*e as f64).powi(2)).sum::()) + .sqrt(); + metrics.insert( + name.into(), + json!({"rows": m, "different": different, "max_abs": max, "relative_l2": l2}), + ); + for (target, p) in order.iter().zip(actual) { + let i = target["request_row"].as_u64().unwrap() as usize; + if name == "option" { + picked[i].push(p); + } else { + first[i] = Some(p); + } + } + } + let heads = Heads::load(&dir.join("heads.safetensors")).unwrap(); + let (zq, u, expected_feats) = (float("zq"), float("u"), float("feats")); + let request: Value = + serde_json::from_slice(&std::fs::read(path("OMNIJEV_ORIGINAL_REQUEST")).unwrap()).unwrap(); + let mut feature_max = 0.0f32; + let mut head_max = 0.0f32; + let (mut hi, mut oi) = (0, 0); + for (group, (_, question)) in layout["groups"] + .as_array() + .unwrap() + .iter() + .zip(request["questions"].as_object().unwrap()) + { + let indices: Vec<_> = group + .as_array() + .unwrap() + .iter() + .map(|i| i.as_u64().unwrap() as usize) + .collect(); + let ps: Vec<_> = indices.iter().map(|&i| picked[i].clone()).collect(); + let fs: Vec<_> = indices.iter().map(|&i| first[i]).collect(); + let feats = heads::lm_features(&ps, &fs); + for (&i, feature) in indices.iter().zip(&feats) { + for (a, e) in feature.iter().zip(&expected_feats[i * 6..(i + 1) * 6]) { + feature_max = feature_max.max((a - e).abs()); + } + } + let q = &zq[indices[0] * 2560..(indices[0] + 1) * 2560]; + let options: Vec<_> = indices + .iter() + .map(|&i| u[i * 2560..(i + 1) * 2560].to_vec()) + .collect(); + let kind = match question["type"].as_str().unwrap() { + "choice" => Kind::Choice, + "noul" => Kind::Noul, + "score" => Kind::Score, + other => panic!("{other}"), + }; + let (actual, key) = if kind == Kind::Score { + let key = format!("ord.{oi}"); + oi += 1; + (heads.ordinal(q, &options).unwrap(), key) + } else { + let key = format!("head.{hi}"); + hi += 1; + ( + heads + .option_probabilities(&options, q, kind.type_id(), &feats) + .unwrap(), + key, + ) + }; + let expected = float(&key); + assert_eq!(actual.len(), expected.len()); + for (a, e) in actual.iter().zip(expected) { + head_max = head_max.max((a - e).abs()); + } + } + assert!(feature_max <= 2e-5, "features maxabs {feature_max}"); + assert!(head_max <= 1e-5, "same-hidden heads maxabs {head_max}"); + metrics.insert( + "features".into(), + json!({"shape": [35,6], "max_abs": feature_max}), + ); + metrics.insert( + "heads".into(), + json!({"questions": 12, "max_abs": head_max}), + ); + println!("{}", Value::Object(metrics.clone())); + if let Some(path) = std::env::var_os("OMNIJEV_LM_RESULT") { + std::fs::write(path, serde_json::to_vec_pretty(&metrics).unwrap()).unwrap(); + } +} + +#[test] +#[ignore = "requires CUDA, OMNIJEV_REFERENCE_EXPORT, OMNIJEV_CUDA_LIB and original complete OMNIJEV_ORIGINAL_REQUEST; CUA_S1_GRAPH=0"] +fn reference_whole_request_outputs_and_recovery() { + let path = |key| PathBuf::from(std::env::var_os(key).expect(key)); + let (mut executor, processor) = Executor::load_one_shot_reference( + &path("OMNIJEV_REFERENCE_EXPORT"), + &path("OMNIJEV_CUDA_LIB"), + ) + .unwrap(); + let raw = std::fs::read(path("OMNIJEV_ORIGINAL_REQUEST")).unwrap(); + let mut prepared = processor.prepare(&raw).unwrap(); + assert_eq!(prepared.questions.len(), 12); + assert_eq!(prepared.layout.prefix, 581); + assert_eq!(prepared.layout.processed_tokens, 1214); + let outputs = executor + .head_outputs(&prepared, false) + .expect("all three types need one-shot outputs"); + assert_eq!(outputs.len(), 12); + for (q, output) in prepared.questions.iter().zip(&outputs) { + assert_eq!(output.len(), q.keys.len()); + assert!(output.iter().all(|p| p.is_finite())); + } + assert!(executor.head_outputs(&prepared, true).is_err()); + let saved = prepared.inputs[0].rows[0].targets[0]; + prepared.inputs[0].rows[0].targets[0].1 = u32::MAX; + assert!(executor.head_outputs(&prepared, false).is_err()); + prepared.inputs[0].rows[0].targets[0] = saved; + assert_eq!(outputs, executor.head_outputs(&prepared, false).unwrap()); + let result = executor.execute(&prepared).unwrap(); + assert_eq!(result.len(), 12); + assert!(result.iter().all(Value::is_object)); +} + /// Real original #114 regression: compilation always sees all twelve questions. /// The externally supplied request/oracle stay private and hash-pinned by the /// validation recipe; no source images or checkpoint tensors are checked in. @@ -37,8 +255,7 @@ fn unmerged_reference_original_q2_gate() { .iter() .position(|q| q.id == "q2") .unwrap(); - assert!(executor.execute(&prepared).is_err()); - assert!(executor.execute_rows(&prepared).is_err()); + assert!(executor.head_outputs(&prepared, true).is_err()); assert!( executor .score_one_shot(&prepared, prepared.questions.len()) diff --git a/tests/qwen3_5/readout.rs b/tests/qwen3_5/readout.rs index 55ab811f..b18d0004 100644 --- a/tests/qwen3_5/readout.rs +++ b/tests/qwen3_5/readout.rs @@ -14,6 +14,24 @@ use omni_qwen3_5_native::model::Model; use safetensors::SafeTensors; use serde_json::Value; +#[test] +fn owned_bf16_hidden_validates_shape_and_values() { + use omni_qwen3_5_native::model::Bf16HiddenRows; + assert!(Bf16HiddenRows::new(0, vec![]).is_err()); + assert!(Bf16HiddenRows::new(2, vec![bf16::ZERO]).is_err()); + assert!(Bf16HiddenRows::new(1, vec![bf16::NAN]).is_err()); + assert!(Bf16HiddenRows::new(1, vec![bf16::INFINITY]).is_err()); + let empty = Bf16HiddenRows::new(2, vec![]).unwrap(); + assert_eq!(empty.rows(), 0); + assert!(empty.row(0).is_err()); + let values = vec![bf16::ZERO, bf16::ONE, -bf16::ONE, bf16::from_f32(2.)]; + let owned = Bf16HiddenRows::new(2, values.clone()).unwrap(); + assert_eq!(owned.rows(), 2); + assert_eq!(owned.row(0).unwrap(), &values[..2]); + assert_eq!(owned.row(1).unwrap(), &values[2..]); + assert!(owned.row(2).is_err()); +} + fn checkpoint() -> PathBuf { PathBuf::from(std::env::var_os("QWEN3_5_CHECKPOINT").expect("set QWEN3_5_CHECKPOINT")) } @@ -262,6 +280,7 @@ fn windows_with_captures_equal_one_pass() { #[test] #[ignore = "requires CUDA, an unmerged OmniJev export and QWEN3_5_ADAPTER; run serially with CUA_S1_GRAPH=0"] fn reference_selected_rows_preserve_full_row_and_recover() { + use omni_qwen3_5_native::model::{Bf16HiddenRows, MAX_REFERENCE_LM_ROWS, ReferenceLmTarget}; let library = PathBuf::from(std::env::var_os("CUA_S1_CUDA_LIB").unwrap()); let adapter = PathBuf::from(std::env::var_os("QWEN3_5_ADAPTER").unwrap()); let mut model = Model::load_with_lora(&checkpoint(), &library, &adapter, 2.).unwrap(); @@ -272,6 +291,54 @@ fn reference_selected_rows_preserve_full_row_and_recover() { let read = model .forward_multimodal_reference_readout(&original.input(), &[t - 1, 10, t - 1, 0], &[]) .unwrap(); + let owned = model + .forward_multimodal_reference_hidden(&original.input(), &[t - 1, 10, t - 1, 0]) + .unwrap(); + let projected = model + .reference_lm_logprobs(&[ + ReferenceLmTarget { + hidden: &owned, + row: 0, + token: 1000, + }, + ReferenceLmTarget { + hidden: &owned, + row: 2, + token: 1000, + }, + ]) + .unwrap(); + assert_eq!(projected[0], projected[1]); + assert!(model.reference_lm_logprobs(&[]).unwrap().is_empty()); + for (row, token) in [(owned.rows(), 1000), (0, u32::MAX)] { + assert!( + model + .reference_lm_logprobs(&[ReferenceLmTarget { + hidden: &owned, + row, + token + }]) + .is_err() + ); + } + let wrong = Bf16HiddenRows::new(1, vec![bf16::ZERO]).unwrap(); + assert!( + model + .reference_lm_logprobs(&[ReferenceLmTarget { + hidden: &wrong, + row: 0, + token: 1000 + }]) + .is_err() + ); + let over: Vec<_> = (0..=MAX_REFERENCE_LM_ROWS) + .map(|_| ReferenceLmTarget { + hidden: &owned, + row: 0, + token: 1000, + }) + .collect(); + assert!(model.reference_lm_logprobs(&over).is_err()); assert_eq!(read.hidden[0], last); assert_eq!(read.hidden[0], read.hidden[2]); assert!(read.logprobs.is_empty()); @@ -304,6 +371,34 @@ fn reference_selected_rows_preserve_full_row_and_recover() { let again = model .forward_multimodal_reference_readout(&original.input(), &[t - 1, 10, t - 1, 0], &[]) .unwrap(); + for (i, expected) in read.hidden.iter().enumerate() { + assert_eq!( + owned + .row(i) + .unwrap() + .iter() + .map(|v| v.to_f32()) + .collect::>(), + *expected + ); + } + assert_eq!( + projected, + model + .reference_lm_logprobs(&[ + ReferenceLmTarget { + hidden: &owned, + row: 0, + token: 1000 + }, + ReferenceLmTarget { + hidden: &owned, + row: 2, + token: 1000 + }, + ]) + .unwrap() + ); assert_eq!( bits(&again.hidden), bits(&read.hidden), From 0e69d34cb3e30c51d1bbd76477bb63204b3bed40 Mon Sep 17 00:00:00 2001 From: psv666 <2693925048@qq.com> Date: Fri, 20 Nov 2026 14:32:14 +0800 Subject: [PATCH 4/4] omnijev: document all-type reference profile and acceptance limits --- README.md | 2 +- docs/architecture.md | 12 +++++- docs/supported-models.md | 2 +- recipe/README.md | 4 +- recipe/omnijev/README.md | 46 +++++++++++++++------ recipe/omnijev/native.md | 79 ++++++++++++++++++++++++++++++++++-- src/models/omnijev/README.md | 52 +++++++++++++++++++----- tests/omnijev/worker.rs | 18 +++++--- 8 files changed, 178 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 1522443b..61e8352e 100644 --- a/README.md +++ b/README.md @@ -193,7 +193,7 @@ checkpoint. Public231 concurrency-1 latency for heads `1ba07d1`, `f968e576`, | autotrust/JEV-27B-VL | [Experimental Rust/CUDA worker](recipe/jev_vl/README.md); single-question text, online inline PNG/JPEG and offline-preencoded image inputs; [validation scope and limits](recipe/jev_vl/validation.md) | | CLM-v0.1-8B | [External worker with a CPU stub encoder](recipe/clm/README.md); contract checks only, real Qwen3-8B decisions unverified by this recipe | | LFM2.5-350M | [Transformers worker](recipe/lfm2/README.md); text state and independent choice questions | -| OmniJev-4B v1.1 | [Native Rust/CUDA worker](recipe/omnijev/native.md), eager: one image, Choice, Noul and Score; [reference comparison on sm_89](recipe/omnijev/native.md#comparison-with-the-reference) | +| OmniJev-4B v1.1 | [Native Rust/CUDA worker](recipe/omnijev/native.md), one image, Choice/Noul/Score; merged default and explicit [unmerged one-shot profile](recipe/omnijev/native.md#explicit-unmerged-one-shot-profile), whose full numerical acceptance remains incomplete; [historical sm_89 comparison](recipe/omnijev/native.md#comparison-with-the-reference) | | Kev-4B | [Pinned CPU export and Rust pointer head](recipe/kev/README.md); no worker or full-model parity yet ([#27](https://github.com/InferMatrixLab/system1-omni/issues/27)) | | D3 (Decision 3.0) | [Python reference worker](recipe/d3/reference.md) on the checkpoint's pinned runtime; `choice`/`noul`/`score` over text, image and video state; author-run on A100 for text and media serving (see the integration and media PRs) | | Vela 2.0 0.3B | [Python reference worker](recipe/vela/reference.md) on the checkpoint's pinned runtime; all five kinds (`choice`/`noul`/`score`/`span`/`set`) for text state; CPU FP32 validated (author-run, evidence in the integration PR) | diff --git a/docs/architecture.md b/docs/architecture.md index 70837b92..07d7b0da 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -144,13 +144,21 @@ call. The Cua-S1 text worker uses it ([#188](https://github.com/InferMatrixLab/system1-omni/issues/188)). -OmniJev's separate unmerged one-shot Score library profile uses +OmniJev's separate unmerged one-shot profile uses `forward_multimodal_reference_readout`: one complete unpadded multimodal row, the existing reference norm/projection/attention and FP32 GDN layer loop, then selected final-norm hidden rows in caller order. It synchronizes before returning or reusing scratch, including error paths. It creates no prefix state and rejects Graph and LM targets; the fixed/window/reuse guards above remain in effect. -This is a bounded library capability, with full worker/all-type parity unverified. +The complete-request executor owns selected BF16 hidden values independently of +scratch and supplies explicit projection groups through `ReferenceLmTarget` / +`reference_lm_logprobs`. OmniJev maps request chunks, candidate ownership and +option tokens; Qwen executes each declared ordered group as one full-vocabulary +GEMM without inferring question semantics or splitting M. It normalizes with the +existing measured CPU FP64 reduction, an approximation to reference FP32. Both +streams drain before releasing admission, including errors. The optional +`OMNIJEV_PROFILE=unmerged-reference` worker uses the existing all-type warmup and +serial scheduler. Full numerical parity remains a separate acceptance gate. See the [OmniJev preparation recipe](../recipe/omnijev/README.md#separate-unmerged-reference-artifact). ## Native worker boundaries diff --git a/docs/supported-models.md b/docs/supported-models.md index f7b707f6..e4bc2163 100644 --- a/docs/supported-models.md +++ b/docs/supported-models.md @@ -27,7 +27,7 @@ Models that are being added are also tracked in issues labeled [new model](https | CLM-v0.1-8B | [External `clm-serve` recipe](../recipe/clm/README.md) with a CPU stub embeddings server | **Stub-encoder contract checks only** ([#23](https://github.com/ThinkFlowLab/system1-omni/pull/23)); not real Qwen3-8B decisions | Real encoder unverified by the merged recipe | Unverified | Python, upstream CLM and head checkpoint; a real encoder requires a separate embeddings server | | LFM2.5-350M | [Transformers choice worker](../recipe/lfm2/README.md) | Tiny-model regression tests; full checkpoint unverified | L40S FP16 [recorded execution checks](../recipe/lfm2/RESULTS.md) | Unverified | Python 3.12, [pinned dependencies](../src/models/lfm2/requirements.lock.txt), explicit candidate batch size | | CLM-v0.1-8B | [Native Rust engine](../src/models/clm/README.md), `clm-run`, in front of a Qwen3-8B `/v1/embeddings` server | Engine-only agreement **4.5e-06** on identical vectors ([#29](https://github.com/ThinkFlowLab/system1-omni/pull/29), [validation](../recipe/clm/native/VALIDATION.md)); contract checks documented ([#23](https://github.com/ThinkFlowLab/system1-omni/pull/23)) | **Validated on compute capability 8.9** ([#29](https://github.com/ThinkFlowLab/system1-omni/pull/29), [validation](../recipe/clm/native/VALIDATION.md)) | Unverified | Rust to build and a separate Qwen3-8B `/v1/embeddings` server for the encoder; the engine itself has no device-specific code | -| OmniJev-4B v1.1 | [Native Rust/CUDA worker](../recipe/omnijev/native.md) on the shared Qwen3.5 vision and language path, eager | Contract, preparation, pixels, heads and finishing checked against the reference; no CPU inference | Compared with the reference on compute capability 8.9 for [140 answers](../recipe/omnijev/native.md#comparison-with-the-reference) | Not supported | Compute capability 8.0 or newer, CUDA toolkit to build, the export from `export.py` | +| OmniJev-4B v1.1 | [Native Rust/CUDA worker](../recipe/omnijev/native.md), merged default or explicit [unmerged one-shot profile](../recipe/omnijev/native.md#explicit-unmerged-one-shot-profile), one image and all three types | Contract, preparation, pixels, heads and finishing checked against the reference; no CPU inference | Historical compute capability 8.9 [140-answer comparison](../recipe/omnijev/native.md#comparison-with-the-reference); L20 unmerged same-hidden LM/features checked, full numerical acceptance incomplete | Not supported | Compute capability 8.0 or newer, CUDA toolkit and appropriate reference operators for the selected profile; matching export from `export.py` | | D3 (Decision 3.0) | [Python reference worker](../recipe/d3/reference.md) on the checkpoint's pinned runtime — text, image and video state over `choice`/`noul`/`score`, with the deliberate transport differences documented in the recipe; the native worker is in progress ([#136](https://github.com/ThinkFlowLab/system1-omni/issues/136)) | Unverified | Author-run on NVIDIA A100 (sm_80): serving smoke and reference parity for text and media (exact decisions, max delta 0.0); evidence in the integration and media PRs. Owner: @MrDongsls. | Unverified | Python 3.12, the [pinned runtime requirements](../recipe/d3/requirements.txt), the [pinned package](https://huggingface.co/vllm-sr/d3) (~52 GB); [media budgets and the 64 MiB total request cap](../recipe/d3/reference.md) | | Vela 2.0 0.3B | [Python reference worker](../recipe/vela/reference.md) running the checkpoint's pinned runtime | **CPU FP32 validated** (author-run, x86 WSL2, 24 cores): startup with all-kind warmup, the four example requests plus five synthetic rejections, and exact-match reference parity against the pinned upstream server on all nine (four 200s byte-equal incl. key order, agreed 422/413); byte-identical frontend forwarding; evidence in the integration PR. Owner: @MrDongsls. | Unverified (bf16-autocast is an untested deployment option) | Not supported | Python 3.12, the [pinned requirements](../recipe/vela/requirements.txt), the [pinned package](https://huggingface.co/vllm-sr/Vela-2.0-0.3B) (1.24 GB FP32 weights; ~4.4 GB with ONNX/assets) | | Vela 2.0 0.8B hybrid | [Python reference worker](../recipe/vela/reference.md) running the checkpoint's pinned runtime (same wire layer as 0.3B) | Same surface as 0.3B plus the `span_heads` field (router/broad dispatch); **CPU FP32 parity** (author-run; evidence in the integration PR) | Unverified (bf16-autocast is the evaluated upstream setting but untested here) | Not supported | Python 3.12, the [pinned requirements](../recipe/vela/requirements.txt), the [pinned package](https://huggingface.co/vllm-sr/Vela-2.0-0.8B) (2.02 GB shard + 4.5 MB broad head; ~3 GB FP32 parameters) | diff --git a/recipe/README.md b/recipe/README.md index fd1b651e..b1be4d3d 100644 --- a/recipe/README.md +++ b/recipe/README.md @@ -45,7 +45,9 @@ For a first real decision, follow the [complete CPU walkthrough](../docs/getting image decisions; bounded H800 validation with explicit deployment limits. - [OmniJev-4B v1.1 preparation](omnijev/README.md): export the checkpoint and regenerate the CPU fixtures; [native worker](omnijev/native.md): serve it on CUDA - and compare it with the reference. + and compare it with the reference. The explicit unmerged one-shot profile + supports complete Choice/Noul/Score requests; full numerical acceptance remains + incomplete. - [Kev-4B preparation](kev/README.md): export all LoRA projections and validate the Rust FP32 pointer head on CPU; worker integration remains pending. - [CLM behind the frontend](clm/README.md): run CLM's own server behind the frontend on diff --git a/recipe/omnijev/README.md b/recipe/omnijev/README.md index b7f864e0..61a6811e 100644 --- a/recipe/omnijev/README.md +++ b/recipe/omnijev/README.md @@ -59,7 +59,7 @@ under a minute. ### Separate unmerged reference artifact -For the eager one-shot Score **library API**, prepare a separate directory: +For the eager one-shot unmerged profile, prepare a separate directory: ```bash .venv-omnijev/bin/python recipe/omnijev/export.py --base qwen3.5-4b \ @@ -71,17 +71,19 @@ This writes `omnijev-unmerged-reference/1`: the original BF16 base language weights, resized tied embedding and saved special rows, plus all 400 separate FP32 rank-32 adapter tensors at alpha 64 / scale 2. Vision, heads, tokenizer and calibration retain the same identity. It does not merge or overwrite an existing -artifact. Both loaders reject the other format; the merged worker does not serve -this artifact. +artifact. Both loaders reject the other format. Select the matching worker +profile explicitly as described in the [native recipe](native.md#explicit-unmerged-one-shot-profile). -Use `Executor::load_one_shot_reference` and `score_one_shot(&prepared, index)` -after preparing the **complete original request**. Keep every question in the +Use `Executor::load_one_shot_reference` and `execute(&prepared)` after preparing +the **complete original request**. `execute_rows` and +`head_outputs(&prepared, false)` also support all three types. Keep every question in the processor input, since selecting a question before compilation changes its longest common prefix. The returned selected hidden rows, ordinal probabilities -and calibrated answer support boundary validation. This API supports one Score -question at a time; whole-request execution, Choice/Noul, LM targets, prefix -reuse and Graphs are rejected. Set `CUA_S1_GRAPH=0`. There is no worker switch -for this incomplete all-type profile. +and calibrated answer from `score_one_shot(&prepared, index)` remain available for +one selected Score's boundary validation. Prefix reuse and language Graphs are +rejected. Set `CUA_S1_GRAPH=0`. Whole-request LM groups include Score rows; +this profile limits all targets together to 16,384 and validates that budget +before dispatch. It fixes the official default chunk size at 64 candidate rows. The root real-weight checks are opt-in and must run serially on a reserved GPU: @@ -101,15 +103,33 @@ cargo test --release --locked -p omni-qwen3-5-native --test readout \ 1214, mode rejection, repeated hidden/head outputs, the Score decision, and every returned probability/confidence at the original 0.01 gate. Optional `OMNIJEV_REFERENCE_RESULT` saves its selected hidden/head output outside the repo. -Passing this bounded check does not establish all-type/full-280, JPEG or HTTP -parity; those require separate complete execution evidence. +Passing this bounded check does not establish full-280 or JPEG parity. + +`reference_whole_request_outputs_and_recovery` uses the same original request +and checks all-type cardinality, repeated raw head outputs, reuse rejection, +an invalid target followed by recovery, and whole-request finishing. +`reference_actual_lm_groups_and_features` additionally consumes the external +`OMNIJEV_LM_CAPTURE` safetensors and `OMNIJEV_LM_LAYOUT` JSON from an actual +official capture of that complete request. The capture hash and the layout's +original-request identity are checked by the test; +the private original image and checkpoint tensors are not committed. It runs +the production Rust 112-row/31-row projections and compares picked +log-probabilities (absolute 1e-5), all six features (2e-5), and all twelve +same-hidden head outputs (1e-5) against the official branch capture. Set +`OMNIJEV_REFERENCE_EXPORT`, `OMNIJEV_CUDA_LIB` and `OMNIJEV_ORIGINAL_REQUEST` as +above, then name that test with `--ignored --exact --nocapture --test-threads=1`. +`OMNIJEV_LM_RESULT` optionally saves diagnostics outside the source tree. +The Qwen selected-row test also checks owned-hidden lifetime across forwards and +scratch growth, duplicate targets, empty groups, bad width/row/token and the group +row budget. The old fixed/window/FP32GDN guards remain. On one L20, the original full-context PNG `video.png/q2` check passes: selected `zq` and three `u` rows exactly match the pinned official unmerged plain-row control; native confidence is 0.5640 against the normative original branch's 0.5613, within the unchanged 0.01 gate. The branch oracle remains authoritative; -the plain-row control only verifies this library path. This coverage is limited -to that observed Score case and the selected-row state/recovery tests. +the plain-row control only verifies that path. The whole-request and same-hidden +LM tests above cover the subsequent all-type implementation; original branch +numerical acceptance still needs the complete original corpus. ### Ordinary CPU fixtures diff --git a/recipe/omnijev/native.md b/recipe/omnijev/native.md index 77eb9a31..2ec9638d 100644 --- a/recipe/omnijev/native.md +++ b/recipe/omnijev/native.md @@ -104,6 +104,80 @@ Malformed JSON gets 400, a body over 12 MiB 413, and a request the contract refu at most 16, preparing, waiting or running; beyond that it answers 503 with `Retry-After: 1`. +## Explicit unmerged one-shot profile + +Prepare the [separate unmerged artifact](README.md#separate-unmerged-reference-artifact), +and build the CUDA library with the reference vision requirements above. The +library must also provide the existing reference language/FP32 GDN operators. +From the repository root: + +```sh +cargo build --release --locked -p omni-omnijev-native -p omni-jev +OMNIJEV_PROFILE=unmerged-reference \ +OMNIJEV_MODEL=$PWD/omnijev-unmerged-reference \ +OMNIJEV_CUDA_LIB=$PWD/target/release/libqwen3_5_cuda.so \ +CUA_S1_GRAPH=0 CUA_S1_VISION_GRAPH=0 \ + target/release/omni-omnijev-native +``` + +`OMNIJEV_PROFILE` accepts `merged-fixed` (default) or `unmerged-reference`; +unknown values fail startup. The latter requires the matching unmerged export, +selects reference vision and language arithmetic, and supports complete +Choice/Noul/Score requests through the existing direct-worker and Rust frontend +interfaces. Its all-type warmup finishes before listening or reporting ready. +Use the frontend launch and request examples above unchanged. Language Graphs +and prefix reuse are rejected. This experimental profile has no speed claim; +full numerical acceptance and JPEG fidelity remain separate gates. + +The backbone runs normal independent full rows. LM projections retain the +official default grouping: chunks of 64 complete-request candidate rows, +option-token rows in order, then a separate zq/first-token group. Score rows +remain in the projection layout. Complete-request targets, including those +unused Score targets, are limited to 16,384 and rejected during preparation +with HTTP 422 above that bound. This is stricter than the merged profile's +Score-exempt LM budget. All logits use the exact 248,079-entry denominator. +Readout/window guards remain; this mode uses no continuation state. + +On actual original-request hidden, native grouped BF16 logits match the +official 112-row and 31-row outputs exactly. The retained CPU FP64 reduction +differs from official FP32 picked log-probabilities by at most 9.54e-7; +the six features differ by at most 1.53e-5 and same-hidden heads by 2.39e-7. +Those controls isolate the LM boundary; they do not replace the original +branch's complete-answer 0.01 and protected-decision gate. + +On one L20 with the pinned BF16 unmerged weights and runtime, the complete +original 28-request/280-answer replay remains **failed** under that unchanged +gate. Raw values round back to the production results on every answer: + +| Input population | Answers within 0.01 (raw and returned) | Protected disagreements | Maximum raw field drift | +| --- | ---: | ---: | ---: | +| PNG | 125/140 | 0 | 0.02967435 | +| JPEG | 54/140 | 3 | 0.18771529 | +| Complete corpus | 179/280 | 3 | 0.18771529 | + +All 140 PNG decisions agree; fifteen PNG answers still exceed 0.01. The largest +is the original complete-context `video.png/q6`. Protected JPEG disagreements +remain `board.jpg/q8`, `arcade.jpg/long_start`, and `web.jpg/one_token`. +The largest returned-field drift is 0.1878. This is a fresh unmerged candidate +run, separate from the earlier merged/reference-vision result above; do not +combine their metrics or infer a uniform improvement. + +For that q6 case, native zq/u equal every BF16 value in a fresh official +plain-full-row control, while differing from the normative cached branch +(maximum hidden delta 0.0625). Official LM/head execution on the identical +native hidden differs from the native head by only 5.96e-8. This locates a +remaining boundary before LM feature consumption; the first differing internal +backbone operator has not been traced. A separate branch-topology stage is +needed, with the original oracle and gates retained. + +The real worker completed all-type warmup before readiness, served the original +complete request as its first request, and matched both the direct core replay +and Rust frontend results exactly except timing. Malformed JSON (400), invalid +questions/model and the reference LM budget (422), and body limit (413) matched +through both HTTP paths; a valid request after errors retained the same result. +These serving checks do not pass the numerical gate. Other GPUs, Metal, +reference reuse/Graphs, and performance remain unverified. + ## Tests The CPU tests need only the committed fixtures; the GPU tests need the export and the @@ -118,9 +192,8 @@ QWEN3_5_CHECKPOINT=$PWD/omnijev-export CUA_S1_CUDA_LIB=$PWD/target/release/libqw ``` These commands select the merged worker's tests. The separate unmerged one-shot -Score library tests require their own artifact and Graph-off settings; see the -[preparation recipe](README.md#separate-unmerged-reference-artifact). They do not -establish a replacement HTTP worker profile. +tests require their own artifact and Graph-off settings; see the +[preparation recipe](README.md#separate-unmerged-reference-artifact). The direct OmniJev window-Graph regression selects eager and Graph models without changing process-wide environment variables. It compares raw FP32 head bits and diff --git a/src/models/omnijev/README.md b/src/models/omnijev/README.md index 10f6d8bf..670b587a 100644 --- a/src/models/omnijev/README.md +++ b/src/models/omnijev/README.md @@ -11,16 +11,18 @@ regenerates the fixtures; [the worker recipe](../../../recipe/omnijev/native.md) runs and validates the worker. An explicit `omnijev-unmerged-reference/1` export and -`Executor::load_one_shot_reference` / `score_one_shot` library API preserve the -separate FP32 LoRA arithmetic for one selected Score question from a complete -prepared request. They reuse the normal full-row Qwen reference execution and -gather the specified final-norm hidden rows, synchronizing both streams before -returning, including errors. The API owns no continuation state. Graphs, reuse, -LM targets and all-question worker execution are unsupported and rejected; -the existing merged worker/defaults remain available. See the +`Executor::load_one_shot_reference` preserve separate FP32 LoRA arithmetic for +complete Choice/Noul/Score requests. Each candidate uses normal full-row Qwen +reference execution; selected BF16 hidden rows are independently owned until the +complete request chunk's LM projections finish. Both streams synchronize on +success and error. `execute` and `execute_rows` use this eager path; +`head_outputs(prepared, true)` still rejects prefix reuse. The narrower +`score_one_shot` diagnostic API remains. `OMNIJEV_PROFILE=unmerged-reference` +selects this experimental worker profile; the default is `merged-fixed`. +Language Graphs and prefix reuse remain unsupported. See the [separate export and validation instructions](../../../recipe/omnijev/README.md#separate-unmerged-reference-artifact). -This bounded library capability does not establish completion of #114's original -all-type 280-answer numerical gate, JPEG fidelity or HTTP acceptance. +This capability does not establish completion of #114's original 280-answer +numerical gate or JPEG fidelity. ## Pinned artifacts @@ -116,7 +118,10 @@ marker, `zq` at the token before its option marker, and, for the LM features, ea option-text token with the position that predicts it. Rotary positions are Qwen3.5's T/H/W positions (the shared `inputs::image_positions`); after the prefix, rows continue from its largest position plus one. Since every row equals a plain forward over prefix -plus row up to rounding, execution may choose its own split points. +plus row up to rounding, the legacy fixed profile chooses its own split points. +The reference profile preserves complete-request LM grouping, while its normal +full-row backbone still differs numerically from the original cached branch +topology. The original branch remains the acceptance oracle. ## Execution @@ -157,6 +162,33 @@ question state `zq` is the first row's, as in the reference. ## Heads and finishing +For the explicit unmerged profile, `processing::reference_lm_chunks` maps every +candidate in question/option order to chunks of 64 request rows, including Score +rows whose ordinal head ignores LM features. Within each chunk it gathers every +option token's preceding hidden row, then projects all of them in one GEMM. A +second GEMM projects the nonempty options' zq rows for their first-token features. +The vocabulary is exactly 248,079; no padded logit participates in normalization. +`Bf16HiddenRows` owns CPU BF16 values beyond subsequent forwards/scratch +growth, and `ReferenceLmTarget` declares the complete projection order. The +generic executor interprets rows/tokens only; OmniJev owns candidate and feature +semantics. The initial selected-readout API continues to reject inline LM targets. + +The existing CPU FP64 log-softmax reduction is retained as a measured +approximation to the official FP32 operation. On actual original-request hidden, +the 112-row option and 31-row first-token projections match official BF16 logits +exactly; selected log-probabilities differ by at most 9.54e-7, six features by +1.53e-5, and same-hidden head outputs by 2.39e-7. The old fixed/64-target/padded +controls also produce exact logits on these particular inputs; this observation +does not establish arbitrary-shape row independence. Request groups normalize +only after all their chunk outputs exist. Empty options retain the reference's +masked -1e4/group log-softmax policy, including feature-layer biases. + +This profile bounds complete-request LM targets, including unused Score rows, +to 16,384. CPU preparation rejects an excess before dispatch (HTTP 422). A Qwen +projection group has the same row limit and retains its complete M; it never +silently splits a declared group. Input, device logits and host readback buffers +grow only as needed and synchronize before replacement and on every exit. + [`heads.rs`](native/src/heads.rs) runs the trained heads in FP32 on the CPU. Choice and Noul use `OptionScorer`: gated option and question MLPs, the LM features, a per-type temperature, and for Choice a softmax that includes the abstain logit; Noul diff --git a/tests/omnijev/worker.rs b/tests/omnijev/worker.rs index 8e8d892c..61f36af1 100644 --- a/tests/omnijev/worker.rs +++ b/tests/omnijev/worker.rs @@ -43,13 +43,17 @@ fn reference_actual_lm_groups_and_features() { match tensor.dtype() { Dtype::F32 => tensor .data() - .chunks_exact(4) - .map(|b| f32::from_le_bytes(b.try_into().unwrap())) + .as_chunks::<4>() + .0 + .iter() + .map(|b| f32::from_le_bytes(*b)) .collect::>(), Dtype::BF16 => tensor .data() - .chunks_exact(2) - .map(|b| bf16::from_le_bytes(b.try_into().unwrap()).to_f32()) + .as_chunks::<2>() + .0 + .iter() + .map(|b| bf16::from_le_bytes(*b).to_f32()) .collect(), other => panic!("unexpected tensor dtype {other:?}"), } @@ -74,8 +78,10 @@ fn reference_actual_lm_groups_and_features() { 2560, tensor .data() - .chunks_exact(2) - .map(|b| bf16::from_le_bytes(b.try_into().unwrap())) + .as_chunks::<2>() + .0 + .iter() + .map(|b| bf16::from_le_bytes(*b)) .collect(), ) .unwrap();