diff --git a/recipe/vela/native.md b/recipe/vela/native.md index 3c2402dc..f80fd04a 100644 --- a/recipe/vela/native.md +++ b/recipe/vela/native.md @@ -3,8 +3,8 @@ The native implementation for [vllm-sr/Vela-2.0-0.8B](https://huggingface.co/vllm-sr/Vela-2.0-0.8B): the exported-bundle gate, the wire contract, and the token-level -processing — all mirrored from the pinned runtime and checked against -frozen fixtures. This is a library; serving runs through the +processing — checked against the pinned runtime's frozen fixtures +within the native input scope below. This is a library; serving runs through the [reference recipe](reference.md)'s Python worker. Run the commands from the repository root with the same `.venv` and @@ -84,6 +84,28 @@ VELA_NATIVE_BUNDLE="$PWD/weights/vela-2.0-0.8b-native" \ ## Scope and limits +- Each completed ordinary row and span-window row must contain at most + **16,384 tokens**, after the existing state clipping and windowing. + Any oversized row rejects the whole request with status 413 and the + `too_long` detail at `body.questions`, before execution. The cap uses + the rendered row's `n_tokens`, including its parts and all question + blocks; it does not limit the request's summed `usage.input_tokens`. + Multiple valid rows/windows may therefore account for more than + 16,384 input tokens together. State clipping, row ordering, window + overlap, and usage accounting otherwise retain the pinned behavior. +- This is a deliberate restriction of the pinned Python runtime's + input domain. Its minimum assembly budget can admit a longer span + header: `instructions="word " * 20000`, one `PERSON` span label and + state `"s"` produce a 20,085-token row at the model-call boundary. + Native processing rejects that request with 413. Reaching that + reference boundary does not establish successful model inference or + an HTTP result. The bundle's pinned `max_length=16384` is unchanged; + the 0.3B member's 8,192 window and the backbone/tokenizer's 262,144 + position metadata do not define this native admission cap. +- Span questions must target a single part; multi-part span targets + are explicitly rejected with 422. Non-finite JSON is rejected with + the native generic 400 message rather than the Python worker's + value-specific wording. These existing input differences remain. - Implemented: the bundle export + manifest gate, the wire admission, and the token-level processing (state mapping, predict mapping with the validation triples, word units, span-head dispatch, the DEV2.0 diff --git a/src/models/vela/README.md b/src/models/vela/README.md index 9d2c8ab9..f7e793f5 100644 --- a/src/models/vela/README.md +++ b/src/models/vela/README.md @@ -258,7 +258,21 @@ The native implementation lives in [`native/`](native/) as the the long-target window rows. All five frozen token plans (`tests/vela/fixtures/native-plan-*.json`) match field-for-field. +Native processing admits at most **16,384 tokens per completed ordinary +or span-window row**, after the existing state clipping and windowing. +An oversized row rejects the whole request with 413 / `too_long` before +execution. The cap is per row, including its parts and question blocks; +the request's summed `usage.input_tokens` may exceed it. Row order, +clipping, window overlap, and usage otherwise retain the pinned rules. +This deliberately narrows the pinned Python input domain: a sufficiently +long span instruction can reach its model-call boundary with more than +16,384 tokens. That observation does not establish a successful forward +or HTTP response. The original manifest identity remains 16,384; neither +the 0.3B's 8,192 window nor the backbone's 262,144 position metadata +widens native support. Multi-part span targets remain an explicit native +422, and non-finite JSON retains the generic native 400 wording. + The F32 heads, scoring, backbone execution and an HTTP worker are not -implemented. Checks: `cargo test -p omni-vela-native` (26 tests, no +implemented. Checks: `cargo test -p omni-vela-native` (ordinary tests, no weights, no GPU; render parity opt-in with `VELA_NATIVE_BUNDLE`; see [recipe/vela/native.md](../../recipe/vela/native.md)). diff --git a/src/models/vela/native/src/processing.rs b/src/models/vela/native/src/processing.rs index ba4c5817..d705568f 100644 --- a/src/models/vela/native/src/processing.rs +++ b/src/models/vela/native/src/processing.rs @@ -274,3 +274,7 @@ fn to_generic(question: &PredictQuestion) -> GenericQuestion { head_override: question.head.clone(), } } + +#[cfg(test)] +#[path = "../../../../../tests/vela/native_row_limits.rs"] +mod row_limit_tests; diff --git a/src/models/vela/native/src/render.rs b/src/models/vela/native/src/render.rs index 2b8f17b5..5a8e7bf8 100644 --- a/src/models/vela/native/src/render.rs +++ b/src/models/vela/native/src/render.rs @@ -671,7 +671,7 @@ impl RenderContext<'_> { &q_lens, &el_lens, &mut segs, - 64.max(self.max_length - budget), + 64.max(self.max_length.saturating_sub(budget)), &overs, shrink.as_deref(), ) @@ -767,6 +767,12 @@ impl RenderContext<'_> { }); } let n_tokens = p_ids.len() + blocks.iter().map(block_len).sum::(); + // The upstream 64-token budget floor can leave an oversized + // span header or repeated target. Native admission bounds each + // completed ordinary/window row, not the request's usage sum. + if n_tokens > self.max_length { + return Err(too_long(self.max_length)); + } Ok(RenderedRow { p_ids, blocks, diff --git a/tests/vela/native_row_limits.rs b/tests/vela/native_row_limits.rs new file mode 100644 index 00000000..776e13f6 --- /dev/null +++ b/tests/vela/native_row_limits.rs @@ -0,0 +1,237 @@ +//! Weight-free regression tests through the real processor and renderer. +//! A tiny whitespace tokenizer keeps token costs deterministic and offline. + +use std::sync::atomic::{AtomicUsize, Ordering}; + +use super::*; +use crate::contract::RequestError; +use crate::render::{Block, RenderedRow}; +use serde_json::json; + +const CAP: usize = 16_384; + +fn processor() -> Processor { + static NEXT: AtomicUsize = AtomicUsize::new(0); + let path = std::env::temp_dir().join(format!( + "vela-row-limits-{}-{}.json", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + std::fs::write( + &path, + r#"{"version":"1.0","truncation":null,"padding":null,"added_tokens":[],"normalizer":null,"pre_tokenizer":{"type":"WhitespaceSplit"},"post_processor":null,"decoder":null,"model":{"type":"WordLevel","vocab":{"[UNK]":0,"word":1,"state":2},"unk_token":"[UNK]"}}"#, + ) + .unwrap(); + let encoder = Encoder::load(&path).expect("small local tokenizer"); + std::fs::remove_file(path).unwrap(); + Processor { + encoder, + max_length: CAP, + router_label_sets: Vec::new(), + has_broad: true, + halu_desc: "a claim not supported by the context".into(), + } +} + +fn question(kind: &str, words: usize) -> Value { + let mut question = json!({"type": kind, "instructions": "word ".repeat(words)}); + let criteria = match kind { + "span" => Some(json!({"PERSON": "person"})), + "choice" | "set" => Some(json!({"a": "first", "b": "second"})), + "score" => Some(json!(["low", "high"])), + _ => None, + }; + if let Some(criteria) = criteria { + question["criteria"] = criteria; + } + question["over"] = json!("request"); + question +} + +fn body(state_words: usize, questions: Value) -> Map { + json!({ + "model": "vllm-sr/Vela-2.0-0.8B", + "state": {"request": "word ".repeat(state_words)}, + "questions": questions, + }) + .as_object() + .unwrap() + .clone() +} + +fn planned(processor: &Processor, body: &Map) -> Planned { + processor + .plan(body) + .unwrap_or_else(|error| panic!("unexpected rejection: {}", error.body)) +} + +fn rejected(result: Result) { + let error = match result { + Err(error) => error, + Ok(plan) => panic!( + "over-limit request was admitted: ordinary={:?}, windows={:?}", + plan.rows.iter().map(|row| row.n_tokens).collect::>(), + plan.window_rows + .iter() + .map(|row| row.n_tokens) + .collect::>() + ), + }; + assert_eq!(error.status, 413); + assert_eq!( + error.body, + json!({"detail": [{ + "loc": ["body", "questions"], + "msg": "the questions alone do not fit the 16384-token window", + "type": "too_long", + }]}) + ); +} + +fn span_overhead(processor: &Processor) -> usize { + planned(processor, &body(1, json!({"q": question("span", 1)}))).rows[0].n_tokens - 1 +} + +#[test] +fn span_rows_at_and_below_the_cap_are_admitted() { + let processor = processor(); + let overhead = span_overhead(&processor); + for length in [CAP - 1, CAP] { + let plan = planned( + &processor, + &body(1, json!({"q": question("span", length - overhead)})), + ); + assert_eq!(plan.rows.len(), 1); + assert_eq!(plan.rows[0].n_tokens, length); + assert!(plan.window_rows.is_empty()); + } +} + +#[test] +fn a_span_row_one_token_over_the_cap_rejects_the_request() { + let processor = processor(); + let words = CAP + 1 - span_overhead(&processor); + rejected(processor.plan(&body(1, json!({"q": question("span", words)})))); +} + +#[test] +fn a_span_header_larger_than_the_budget_returns_413_without_panicking() { + let processor = processor(); + rejected(processor.plan(&body(1, json!({"q": question("span", 20_000)})))); +} + +#[test] +fn non_span_schema_overflow_keeps_the_whole_request_413() { + let processor = processor(); + for kind in ["noul", "choice", "score", "set"] { + rejected(processor.plan(&body(1, json!({"q": question(kind, 20_000)})))); + } +} + +#[test] +fn mixed_rows_can_exceed_the_cap_in_total_and_keep_question_order() { + let processor = processor(); + let plan = planned( + &processor, + &body( + 1, + json!({ + "topic": question("noul", 1), + "first": question("span", 9_000), + "second": question("span", 9_000), + }), + ), + ); + assert_eq!(plan.rows.len(), 2); + assert!(plan.rows.iter().all(|row| row.n_tokens <= CAP)); + assert!(plan.rows.iter().map(|row| row.n_tokens).sum::() > CAP); + let order: Vec<_> = plan + .plan + .as_array() + .unwrap() + .iter() + .map(|item| item["qid"].as_str().unwrap()) + .collect(); + assert_eq!(order, ["topic", "first", "second"]); + assert!(matches!(&plan.rows[0].blocks[0], Block::Question { qid, .. } if qid == "topic")); + assert!(matches!(&plan.rows[1].blocks[0], Block::Span { .. })); +} + +#[test] +fn an_oversized_later_row_rejects_the_whole_request() { + let processor = processor(); + let words = CAP + 1 - span_overhead(&processor); + rejected(processor.plan(&body( + 1, + json!({ + "first": question("span", 1), + "second": question("span", words), + }), + ))); +} + +#[test] +fn window_rows_can_exceed_the_cap_in_total() { + let processor = processor(); + let plan = planned(&processor, &body(5_000, json!({"q": question("span", 1)}))); + assert_eq!(plan.rows.len(), 1); + assert!(plan.window_rows.len() > 1); + assert_eq!(plan.rows[0].trunc.protected_cut, 0); + assert!( + plan.rows + .iter() + .chain(&plan.window_rows) + .all(|row| row.n_tokens <= CAP) + ); + assert!(plan.window_rows.iter().all(|row| row.window)); + assert!( + plan.rows + .iter() + .chain(&plan.window_rows) + .map(|row| row.n_tokens) + .sum::() + > CAP + ); +} + +#[test] +fn over_limit_raw_state_keeps_the_existing_clipping() { + let processor = processor(); + let plan = planned(&processor, &body(20_000, json!({"q": question("noul", 1)}))); + assert_eq!(plan.rows.len(), 1); + assert!(plan.rows[0].n_tokens <= CAP); + assert!(plan.rows[0].trunc.protected_cut > 0); + assert!(plan.rows[0].seg_lens["user"] < 20_000); +} + +fn primary_row(processor: &Processor, body: &Map) -> RenderedRow { + let state = system_one_state(&body["state"]).unwrap(); + let mapped = system_one_to_predict(state, body["questions"].as_object().unwrap()).unwrap(); + let generic: Vec<_> = mapped.request.questions.iter().map(to_generic).collect(); + let span_roles = ["user".to_string()].into_iter().collect(); + let parts = processor + .prepared_parts(&mapped.request.parts, &span_roles) + .unwrap(); + let context = RenderContext { + encoder: &processor.encoder, + max_length: CAP, + router_label_sets: &processor.router_label_sets, + has_broad: processor.has_broad, + halu_desc: processor.halu_desc.clone(), + }; + let (questions, dispatch) = context.prepare_questions(&generic).unwrap(); + context.render(&parts, &questions, &dispatch).unwrap() +} + +#[test] +fn an_oversized_window_rejects_even_when_the_primary_row_fits() { + let processor = processor(); + let body = body(2_176, json!({"q": question("span", CAP - 85)})); + let primary = primary_row(&processor, &body); + assert!( + primary.n_tokens <= CAP, + "primary row has {} tokens", + primary.n_tokens + ); + rejected(processor.plan(&body)); +}