diff --git a/recipe/vela/example-request-unicode.json b/recipe/vela/example-request-unicode.json new file mode 100644 index 00000000..095f05e4 --- /dev/null +++ b/recipe/vela/example-request-unicode.json @@ -0,0 +1,46 @@ +{ + "model": "vllm-sr/Vela-2.0-0.3B", + "state": { + "request": "😀 客户张伟,邮箱 zhang.wei@example.cn。尾部🚀", + "source": "订单将在三个工作日内审核。", + "answer": "张伟的订单已经完成。" + }, + "questions": { + "pii": { + "type": "span", + "instructions": "Which spans are personal information?", + "criteria": { + "PERSON": "a person's name", + "EMAIL_ADDRESS": "an e-mail address" + }, + "over": "request", + "threshold": 0.05 + }, + "topics": { + "type": "set", + "instructions": "Which topics are mentioned?", + "criteria": { + "contact": "a customer's contact information", + "order": "an order or its processing status" + }, + "threshold": 0.5 + }, + "kind": { + "type": "choice", + "instructions": "What is the customer's message about?", + "criteria": { + "profile": "customer profile or contact information", + "billing": "an invoice or payment" + } + }, + "tone": { + "type": "score", + "instructions": "How formal is the message?", + "criteria": ["casual", "neutral", "formal"] + }, + "billing": { + "type": "noul", + "instructions": "Does the message concern an invoice or payment?" + } + } +} diff --git a/recipe/vela/reference.md b/recipe/vela/reference.md index 559a7afb..0091c064 100644 --- a/recipe/vela/reference.md +++ b/recipe/vela/reference.md @@ -280,6 +280,112 @@ bf16-autocast numbers for the baseline. The tool rewrites the example files' `model` field to the served member and exports the hybrid's oracle fixtures under an `oracle-hybrid-` prefix. +## Contract decisions and acceptance + +This recipe records the Vela-specific policy for +[#169](https://github.com/InferMatrixLab/system1-omni/issues/169), following the +reference workers merged in #151 and #174. The older #137 proposal's +span/set rejection and no-truncation wording does not describe these workers. +The shared contract RFC [#61](https://github.com/InferMatrixLab/system1-omni/issues/61) +remains open; this policy does not prescribe other models' response fields, +confidence or token accounting. + +| Surface | Vela policy | +| --- | --- | +| Question kinds | Forward all five kinds: `choice`, `noul`, `score`, `span`, `set`. Preserve the span question's Noul view and each set label's Noul view together with top-level `spans`, `sets`, `thresholds`, and the hybrid-only `span_heads`. The Rust frontend preserves the worker's status, content type and body. | +| Span identity and offsets | A single state-field string in `over` gives field-local half-open `[start, end)` Unicode-code-point offsets, with clipping when fields share a part. A part name, default target, or list resolving to one part preserves offsets into that rendered part, including field headers; even `["request"]` differs from `"request"` when fields share a part. Keep labels, probabilities, text and the request's target context. The 0.3B has no `span_heads`; hybrids retain their router/broad identity there. | +| Set interpretation | Per-label probabilities are independent. Select every label whose probability is strictly greater than the applied threshold; preserve label order and threshold echoes. No cross-label softmax or single-winner conversion. | +| Length limits | Preserve the upstream state truncation/windowing policy described above. Schema/questions that cannot fit without state return whole-request 413. A successful response does not establish that every state character was examined. | +| Confidence | Choice: top-two probability margin. Score: one minus normalized level variance. These are reference distribution statistics. Learned `tau` / `tau_span` and inference calibration temperatures remain separate. | +| Usage | Sum tokens of every rendered sequence actually evaluated, including additional span rows and windows. Count a single sequence once when there is only one; do not deduplicate repeated state or replace this with a per-question estimate. Output tokens are always zero. | +| Validation | Preserve whole-request errors and the pinned runtime's field and question-walk precedence. Invalid span/set, schema-only overflow and state truncation are different cases. | + +[`example-request-unicode.json`](example-request-unicode.json) puts CJK and +non-BMP characters before a span target while exercising all five kinds. An +acceptance run must retain a positive span after the emoji; otherwise an empty +span list cannot distinguish code-point offsets from UTF-8 bytes or UTF-16 code +units. For every retained span, slicing the named field at `[start:end]` must +produce its `text` for this explicit field-string target. CJK alone distinguishes bytes but does not distinguish UTF-16 +code units from code points. + +The final acceptance has separate gates: + +1. Generate the 0.3B CPU FP32 oracle from the pinned package and compare the + worker pipeline with the upstream server. Keep package/dependency revisions, + complete responses and failures outside the source tree; retain only small + deterministic fixtures under root `tests/`. +2. Check Unicode spans, independent set labels/thresholds, confidence and usage, + invalid span/set and error precedence, schema-only overflow, and the actual + upstream truncation boundary. The ordinary fake-worker tests establish only + the paths they exercise; they cannot establish the runtime's truncation or + model numerics. +3. Check the same successful/error responses directly and through the Rust + frontend, preserving extension fields and answer order. Ignore a timing field + only when the model's comparison protocol explicitly names it; the Vela + reference responses used here have no timing-field exemption. +4. Compare native processing, heads, response finishing and device execution + against the matching member's oracle separately, using its declared + precision/backend tolerance and identical discrete decisions/span structure. + A CPU reference result or a backbone-logit gate alone does not establish + native HTTP support or final answer parity. + +The hash-gated upstream contract checks are opt-in and do not execute a model +forward. They cover invalid span/set and error precedence, named-field Unicode +offsets with controlled head outputs, and the actual upstream 8,192-token +assembly boundary, unprotected/protected cuts and schema-only overflow: + +~~~sh +VELA_CONTRACT_PACKAGE="$VELA_PKG" PYTHONPATH=src \ + .venv/bin/python -m pytest tests/vela/test_runtime_contract.py -q +~~~ + +Without that environment variable these tests skip; ordinary CPU/workspace +checks remain independent of weights. A successful rendering/assembly check is +separate from the real CPU FP32 oracle and native precision/backend comparisons. + +The small Unicode oracle under root `tests/vela/fixtures/` was exported from +the pinned 0.3B CPU FP32 runtime, using Python 3.12.13, torch 2.11.0+cpu and the +other recipe pins, with eight OMP/MKL threads. It retains actual predictions, +including the partial PERSON span; it is not a task-accuracy label. Its positive +EMAIL_ADDRESS span is `[10, 30)` after the emoji, and the set probabilities sum +to more than one. The root fixture tests bind it to the maintained request and +production response gate. Regenerate that one case from the repository root: + +~~~sh +OMP_NUM_THREADS=8 MKL_NUM_THREADS=8 PYTHONPATH=src \ + .venv/bin/python recipe/vela/compare_reference.py \ + --model "$VELA_PKG" --device cpu \ + --fixture recipe/vela/example-request-unicode.json \ + --export tests/vela/fixtures/oracle-example-request-unicode.json +~~~ + +This records worker/upstream reference parity. Direct/frontend forwarding and +native/device comparisons remain separate acceptance checks. + +### Pending contract decisions + +The following require an explicit contract decision; this slice preserves the +existing reference and introduces no external behavior change: + +- Whether #61 formally adopts Vela's extension fields and model-specific + confidence, rendered-row usage and state truncation, and reconciles #137's old + rejection/no-truncation acceptance wording. Discussion favoring model-defined + fields is [recorded in #61](https://github.com/InferMatrixLab/system1-omni/issues/61#issuecomment-6021490614); + it is not a completed shared-contract decision. +- Whether span consumers may use part-relative/list targets or the accepted + contract should require a single named field. The response does not add a new + coordinate-origin field. Cross-part span lists are not qualified by these + fixtures: the pinned upstream preparation raises `TypeError` before a forward, + so their supported scope and client-error status need resolution. This PR does + not turn that failure into an accepted error contract. +- Whether the shared error envelope should replace Vela's reference `detail` + list and other recorded transport differences. Keep current reference errors + until a deliberate change is agreed and tested. +- Final native acceptance remains separate: matching member/oracle, precision, + structural decisions, router/broad dispatch and declared backend tolerances + must be checked by the independently assigned native phase. CPU reference and + forwarding success do not close those gates. + ## Troubleshooting and limits - The package directory must be the pinned revision's download; the @@ -292,9 +398,10 @@ oracle fixtures under an `oracle-hybrid-` prefix. - `/health` and `/ready` are upstream's minimal bodies on both members, not the richer shape of some other workers in this repository. -- Long states are truncated (see Error behavior); if a consumer must - never miss tail text, check `usage.input_tokens` against the expected - budget or enforce limits upstream of the worker. +- Long states are truncated (see Error behavior). `usage.input_tokens` is work + accounting and cannot prove that the complete state was examined, especially + with multiple rows/windows. Consumers requiring full-state guardrails must + enforce an input bound before this worker's truncation path. - GPU bf16-autocast serving (`--device cuda`) is untested in this evidence; treat it as a deployment option to validate separately. - GPU evidence for the supported-models table is recorded in the diff --git a/src/models/vela/README.md b/src/models/vela/README.md index 74f2c4bb..4b35ca8e 100644 --- a/src/models/vela/README.md +++ b/src/models/vela/README.md @@ -31,7 +31,7 @@ implemented. | Runtime | the checkpoint's own flat files: `vela2_inference.py`, `modeling_vela2.py`, `configuration_vela2.py`, `vela2_serve.py` | the same four modules, decoder-specific | | Calibration | the package's own `calibration.json` (never cross-copied between members — shipped thresholds differ) | same rule; the side-loaded broad head needs its `broad_head` block | | License | Apache-2.0; the inherited tokenizer material carries Gemma Terms of Use | Apache-2.0; Qwen3.5-0.8B lineage notices ship in the package | -| Runtime dependencies | [recipe/vela/requirements.txt](../../recipe/vela/requirements.txt) (torch, transformers, safetensors, tokenizers, numpy) | same | +| Runtime dependencies | [recipe/vela/requirements.txt](../../../recipe/vela/requirements.txt) (torch, transformers, safetensors, tokenizers, numpy) | same | The 0.3B package ships no manifest, so its gate checks the config's structural identity (model type, architecture, `max_length` 8,192, the @@ -105,8 +105,12 @@ The response envelope is the pinned runtime's, passed through verbatim formulas are mirrored as pure functions in `postprocess.py` for the CPU contract tests. - Top-level `spans: {qid: [{label, start, end, text, probability}]}` - with Unicode code-point offsets into the targeted field's text - (clipped and re-trimmed to the field when it shares a part), + with Unicode code-point offsets into the targeted field's text when `over` + is a single state-field string (clipped and re-trimmed when it shares a part). + Part/default targets and lists resolving to one part retain offsets into the + rendered part, so field headers count; `["request"]` is not a field-local + equivalent of `"request"` when fields share a part. Cross-part span lists + remain an unresolved validation/error case. The response also preserves `sets: {qid: {"selected", "probabilities"}}` with strict `>` threshold selection, and `thresholds: {qid: t}` echoing the applied value. Tie-breaking inside a question is insertion order (argmax over @@ -202,9 +206,13 @@ every question kind), so answering at all means ready. single-flight lock, the API-key gate and orchestration. - The Rust frontend owns transport and forwarding, not model state. -The server-side error surface and envelope semantics are recorded as -this repository's local stance for Vela 2.0; nothing is proposed into -the shared worker-contract issue, and nothing is posted upstream. +The server-side error surface and envelope semantics are this repository's +local stance for Vela 2.0. The recipe's +[contract decisions and acceptance](../../../recipe/vela/reference.md#contract-decisions-and-acceptance) +reconciles the older proposal with the shipped five-kind reference, including +Unicode offsets, truncation and rendered-row token accounting. The shared +worker-contract RFC remains open; this records Vela's policy without changing +other workers. ## Validation @@ -224,4 +232,4 @@ the shared worker-contract issue, and nothing is posted upstream. flash-linear-attention) is the oracle setting; GPU bf16-autocast is a deployment option outside these slices' evidence. - The serving procedure, hardware requirements and the recorded - evidence pointer are [recipe/vela/reference.md](../../recipe/vela/reference.md). + evidence pointer are [recipe/vela/reference.md](../../../recipe/vela/reference.md). diff --git a/tests/vela/fixtures/oracle-example-request-unicode.json b/tests/vela/fixtures/oracle-example-request-unicode.json new file mode 100644 index 00000000..6c4d2040 --- /dev/null +++ b/tests/vela/fixtures/oracle-example-request-unicode.json @@ -0,0 +1,215 @@ +{ + "case": "example-request-unicode.json", + "request": { + "model": "vllm-sr/Vela-2.0-0.3B", + "state": { + "request": "😀 客户张伟,邮箱 zhang.wei@example.cn。尾部🚀", + "source": "订单将在三个工作日内审核。", + "answer": "张伟的订单已经完成。" + }, + "questions": { + "pii": { + "type": "span", + "instructions": "Which spans are personal information?", + "criteria": { + "PERSON": "a person's name", + "EMAIL_ADDRESS": "an e-mail address" + }, + "over": "request", + "threshold": 0.05 + }, + "topics": { + "type": "set", + "instructions": "Which topics are mentioned?", + "criteria": { + "contact": "a customer's contact information", + "order": "an order or its processing status" + }, + "threshold": 0.5 + }, + "kind": { + "type": "choice", + "instructions": "What is the customer's message about?", + "criteria": { + "profile": "customer profile or contact information", + "billing": "an invoice or payment" + } + }, + "tone": { + "type": "score", + "instructions": "How formal is the message?", + "criteria": [ + "casual", + "neutral", + "formal" + ] + }, + "billing": { + "type": "noul", + "instructions": "Does the message concern an invoice or payment?" + } + } + }, + "system_one": { + "model": "vllm-sr/Vela-2.0-0.3B", + "answers": { + "pii": { + "type": "noul", + "noul": 0.488608729974243 + }, + "topics.contact": { + "type": "noul", + "noul": 0.43475786270781963 + }, + "topics.order": { + "type": "noul", + "noul": 0.9338471738856106 + }, + "kind": { + "type": "choice", + "choice": "profile", + "confidence": 0.5812821969558832, + "probabilities": { + "profile": 0.7906410984779416, + "billing": 0.20935890152205835 + } + }, + "tone": { + "type": "score", + "score": 1.0749506674540588, + "confidence": 0.17860283610656524, + "legend": { + "0": "casual", + "1": "neutral", + "2": "formal" + }, + "probabilities": { + "0": 0.23913252218001999, + "1": 0.4467842881859013, + "2": 0.3140831896340787 + } + }, + "billing": { + "type": "noul", + "noul": 0.7967674039907597 + } + }, + "usage": { + "input_tokens": 155, + "output_tokens": 0 + }, + "spans": { + "pii": [ + { + "label": "PERSON", + "start": 4, + "end": 5, + "text": "张", + "probability": 0.10246830433607101 + }, + { + "label": "EMAIL_ADDRESS", + "start": 10, + "end": 30, + "text": "zhang.wei@example.cn", + "probability": 0.488608717918396 + } + ] + }, + "sets": { + "topics": { + "selected": [ + "order" + ], + "probabilities": { + "contact": 0.43475786270781963, + "order": 0.9338471738856106 + } + } + }, + "thresholds": { + "pii": 0.05, + "topics": 0.5 + } + }, + "predict": { + "answers": [ + { + "id": "pii", + "type": "span", + "target_part": "user", + "spans": [ + { + "start": 4, + "end": 5, + "label": "PERSON", + "probability": 0.10246830433607101, + "text": "张" + }, + { + "start": 10, + "end": 30, + "label": "EMAIL_ADDRESS", + "probability": 0.488608717918396, + "text": "zhang.wei@example.cn" + } + ], + "threshold": 0.05, + "threshold_rule": "given" + }, + { + "id": "topics", + "type": "set", + "answer": [ + "order" + ], + "probabilities": { + "contact": 0.43475786270781963, + "order": 0.9338471738856106 + }, + "threshold": 0.5 + }, + { + "id": "kind", + "type": "choice", + "answer": "profile", + "probabilities": { + "profile": 0.7906410984779416, + "billing": 0.20935890152205835 + }, + "abstain_probability": 0.19304637686297574 + }, + { + "id": "tone", + "type": "score", + "answer": "neutral", + "probabilities": { + "casual": 0.23913252218001999, + "neutral": 0.4467842881859013, + "formal": 0.3140831896340787 + }, + "expected_index": 1.0749506674540588, + "score": 0.5374753337270294 + }, + { + "id": "billing", + "type": "choice", + "answer": "yes", + "probabilities": { + "no": 0.20323259600924032, + "yes": 0.7967674039907597 + }, + "abstain_probability": 0.01893785421896126 + } + ], + "meta": { + "tokens": { + "user": 17, + "context": 8, + "answer": 7 + }, + "windows": 0, + "sequences": 1 + } + } +} diff --git a/tests/vela/test_oracle_fixtures.py b/tests/vela/test_oracle_fixtures.py new file mode 100644 index 00000000..92426e95 --- /dev/null +++ b/tests/vela/test_oracle_fixtures.py @@ -0,0 +1,76 @@ +"""Consume the frozen 0.3B CPU FP32 oracles without weights or torch. + +The Unicode case was recorded through compare_reference.py from the pinned +d6f03aa9 package, with torch2.11.0+cpu, Python3.12.13, and the recipe dependency +pins. Generation and the distinction from native/device evidence are documented +in recipe/vela/reference.md. The original four oracle files are unchanged. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from models.vela.postprocess import finish_response + +ROOT = Path(__file__).resolve().parents[2] +CASES = ["example-request", "example-request-cjk", "example-request-set-threshold", + "example-request-score", "example-request-unicode"] + + +def load_oracle(case): + return json.loads((ROOT / "tests/vela/fixtures" / f"oracle-{case}.json") + .read_text(encoding="utf-8")) + + +@pytest.mark.parametrize("case", CASES) +def test_oracle_request_is_the_maintained_recipe_example(case): + recorded = load_oracle(case) + request = json.loads((ROOT / "recipe/vela" / f"{case}.json").read_text(encoding="utf-8")) + assert recorded["request"] == request + assert list(recorded["request"]) == list(request) + assert Path(recorded["case"]).stem == case + + +@pytest.mark.parametrize("case", CASES) +def test_reference_response_passes_the_production_envelope_gate(case): + recorded = load_oracle(case) + response = recorded["system_one"] + assert response["model"] == recorded["request"]["model"] + assert finish_response(recorded["request"]["questions"], response) is response + + +def test_unicode_oracle_has_positive_spans_after_non_bmp_text(): + recorded = load_oracle("example-request-unicode") + response = recorded["system_one"] + question = recorded["request"]["questions"]["pii"] + field = recorded["request"]["state"][question["over"]] + spans = response["spans"]["pii"] + assert spans, "an empty oracle cannot test offset units" + email = next(span for span in spans if span["label"] == "EMAIL_ADDRESS") + assert (email["start"], email["end"], email["text"]) == (10, 30, "zhang.wei@example.cn") + for span in spans: + start, end = span["start"], span["end"] + assert field[start:end] == span["text"] + assert any(ord(char) > 0xffff for char in field[:start]) + assert len(field[:start].encode("utf-8")) != start + assert len(field[:start].encode("utf-16-le")) // 2 != start + assert response["thresholds"]["pii"] == question["threshold"] == 0.05 + assert "span_heads" not in response # the 0.3B has one span head + + +def test_unicode_set_probabilities_remain_independent_and_keep_noul_views(): + recorded = load_oracle("example-request-unicode") + response = recorded["system_one"] + result = response["sets"]["topics"] + probabilities = result["probabilities"] + assert list(probabilities) == ["contact", "order"] + assert sum(probabilities.values()) > 1.3 # cannot pass after cross-label softmax + threshold = response["thresholds"]["topics"] + assert threshold == recorded["request"]["questions"]["topics"]["threshold"] == 0.5 + assert result["selected"] == [label for label, p in probabilities.items() if p > threshold] + for label, probability in probabilities.items(): + assert response["answers"][f"topics.{label}"] == {"type": "noul", "noul": probability} + assert response["usage"] == {"input_tokens": 155, "output_tokens": 0} diff --git a/tests/vela/test_runtime_contract.py b/tests/vela/test_runtime_contract.py new file mode 100644 index 00000000..8202165c --- /dev/null +++ b/tests/vela/test_runtime_contract.py @@ -0,0 +1,154 @@ +"""Opt-in contract checks against the hash-pinned 0.3B runtime. + +These exercise upstream validation, response assembly with controlled model +outputs, and the actual token-budget function. They do not run a model forward +or establish learned-head/native parity. Ordinary CPU suites skip them without +VELA_CONTRACT_PACKAGE; no package is downloaded by a test. +""" + +from __future__ import annotations + +import importlib +import json +import os +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +@pytest.fixture(scope="module") +def pinned_runtime(): + package = os.environ.get("VELA_CONTRACT_PACKAGE") + if not package: + pytest.skip("set VELA_CONTRACT_PACKAGE to the pinned 0.3B package") + from models.vela.engine import ( + put_first_on_sys_path, + verify_no_foreign_runtime, + verify_package, + ) + + root = Path(package).resolve() + verify_package(root) + verify_no_foreign_runtime(root) + previous_path = sys.path[:] + previous_bytecode = sys.dont_write_bytecode + try: + sys.dont_write_bytecode = True + put_first_on_sys_path(root) + runtime = importlib.import_module("vela2_inference") + finally: + sys.path[:] = previous_path + sys.dont_write_bytecode = previous_bytecode + config = json.loads((root / "config.json").read_text(encoding="utf-8")) + return runtime, config + + +@pytest.mark.parametrize( + "state,questions,location", + [ + (None, {}, ["state"]), + ("valid", {}, ["questions"]), + ("valid", {"bad": {"type": "span", "instructions": "Find.", + "criteria": ["PERSON"]}}, ["questions", "bad", "criteria"]), + ("valid", {"bad": {"type": "set", "instructions": "Select.", + "criteria": []}}, ["questions", "bad", "criteria"]), + ("valid", {"bad": {"type": "set", "instructions": "Select.", + "criteria": {"x": "x"}, "threshold": True}}, + ["questions", "bad", "threshold"]), + ("valid", {"bad": {"type": "span", "instructions": "Find.", + "criteria": [], "over": "missing"}}, + ["questions", "bad", "over"]), + ("valid", {"first": {"type": "set", "instructions": "Select.", "criteria": []}, + "second": {"type": "span", "instructions": "Find.", "criteria": []}}, + ["questions", "first", "criteria"]), + ("valid", {"labels": {"type": "set", "instructions": "Select.", + "criteria": {"a": "a label"}}, + "labels.a": {"type": "noul", "instructions": "Yes?"}}, + ["questions", "labels.a"]), + ], +) +def test_upstream_validation_and_error_precedence(pinned_runtime, state, questions, location): + runtime, _ = pinned_runtime + with pytest.raises(runtime.SystemOneError) as error: + runtime.system_one_to_predict(state, questions) + assert error.value.detail()["loc"] == ["body", *location] + + +@pytest.mark.parametrize("over,field_relative", [("request", True), (["request"], False), ("user", False)]) +def test_upstream_span_coordinate_origin_depends_on_over(pinned_runtime, over, field_relative): + runtime, _ = pinned_runtime + state = {"prompt": "前文🚀", "request": "😀 张伟 来自上海"} + questions = {"pii": {"type": "span", "instructions": "Find names.", + "criteria": {"PERSON": "a person"}, "over": over}} + # Two aliased fields share a user part. This expected prefix is independent + # of system_one_state's calculated field map. + prefix = "prompt:\n前文🚀\n\nrequest:\n" + absolute_start = len(prefix) + 2 + + def predict(request, span_limits): + assert request["parts"] == [{"type": "user", "text": prefix + state["request"]}] + expected_limits = {"pii": (len(prefix), len(prefix) + len(state["request"]))} if field_relative else {} + assert span_limits == expected_limits + return ({"answers": [{"id": "pii", "threshold": 0.5, "spans": [ + {"label": "PERSON", "start": absolute_start, "end": absolute_start + 2, + "text": "张伟", "probability": 0.9}, + ]}]}, {"pii": 0.95}, 37) + + response = runtime.system_one_response(SimpleNamespace(_predict=predict), state, questions) + start = 2 if field_relative else absolute_start + assert response["spans"]["pii"] == [ + {"label": "PERSON", "start": start, "end": start + 2, "text": "张伟", "probability": 0.9}, + ] + coordinate_text = state["request"] if field_relative else prefix + state["request"] + assert coordinate_text[start:start + 2] == "张伟" + assert state["request"][2:4] == "张伟" + assert len(state["request"][:2].encode("utf-8")) != 2 + assert len(state["request"][:2].encode("utf-16-le")) // 2 != 2 + assert response["answers"]["pii"] == {"type": "noul", "noul": 0.95} + assert response["thresholds"] == {"pii": 0.5} + assert response["usage"] == {"input_tokens": 37, "output_tokens": 0} + + +def assemble(runtime, config, segments, *, protected=(), shrink="user", instructions=1): + # Valid two-option question with controlled token IDs: exact sequence + # geometry, independent of tokenizer/model arithmetic. + token = config["bos_token_id"] + question = {"qid": "q", "type": "choice", "over": "user", + "qtoks": [token] * instructions, + "opts": [("a", [token]), ("b", [token])], + "abstain": False, "len": 5 + instructions} + return runtime.assemble(config["marker_ids"], token, config["eos_token_id"], + [question], [], segments, config["max_length"], + set(protected), shrink) + + +def test_upstream_exact_window_boundary_truncates_state(pinned_runtime): + runtime, config = pinned_runtime + limit = config["max_length"] + assert limit == 8192 + # BOS/EOS + SEP + one segment marker + six question tokens. + capacity = limit - 10 + exact = assemble(runtime, config, [("user", [7] * capacity)]) + cut = assemble(runtime, config, [("user", [7] * (capacity + 1))]) + assert len(exact["ids"]) == len(cut["ids"]) == limit + assert exact["trunc"] == {"shrink_cut": 0, "protected_cut": 0} + assert cut["trunc"] == {"shrink_cut": 1, "protected_cut": 0} + assert cut["seg_lens"] == {"user": capacity} + + +def test_upstream_unprotected_floor_precedes_protected_cut(pinned_runtime): + runtime, config = pinned_runtime + result = assemble(runtime, config, + [("context", [7] * 65), ("user", [8] * 8181)], + protected=("user",), shrink="context") + assert len(result["ids"]) == 8192 + assert result["seg_lens"] == {"context": 1, "user": 8180} + assert result["trunc"] == {"shrink_cut": 64, "protected_cut": 1} + + +def test_upstream_schema_alone_overflow_is_distinct(pinned_runtime): + runtime, config = pinned_runtime + with pytest.raises(ValueError, match="schema alone does not fit"): + assemble(runtime, config, [("user", [7])], instructions=8192)