From c66892e2301f71165b588c2a81e4165e44fe80f0 Mon Sep 17 00:00:00 2001 From: IvGolovach <20299097+IvGolovach@users.noreply.github.com> Date: Sat, 12 Sep 2026 02:55:26 -0700 Subject: [PATCH 01/18] feat(skippy): serve and certify non-chat GGUF workloads Add workload-aware model advertisement and routing, local execution for embeddings, rerank, encoder-decoder, OCR, speech synthesis, and speech recognition, and matching OpenAI-compatible endpoints. Keep unsupported split placements fail-closed and preserve additive mixed-version mesh behavior. Certify pinned representatives against independent monolithic llama.cpp oracles, with separate smoke and oracle lanes, bound evidence, native ABI fixes, HTTP/SDK coverage, and user documentation. --- .../manage-ci/references/current-inventory.md | 21 +- Cargo.lock | 19 + ci/ci.md | 2 +- ci/llama-canary/family-certified.json | 156 +++++ ci/llama-canary/fixtures/audio-smoke.wav | Bin 0 -> 29072 bytes .../manifests/competitive-benchmark.json | 2 +- .../manifests/hf-download-smoke.json | 2 +- .../manifests/openai-smoke.json | 2 +- .../manifests/product-integration-smoke.json | 2 +- .../manifests/product-smoke.json | 2 +- ci/model-artifacts/manifests/radix-cache.json | 2 +- .../manifests/safetensors-runtime-smoke.json | 2 +- .../manifests/scripted-binary-smoke.json | 2 +- ci/model-artifacts/manifests/sdk-smoke.json | 2 +- .../manifests/skippy-ci-smoke.json | 2 +- .../manifests/skippy-correctness.json | 2 +- .../manifests/skippy-parity.json | 2 +- ci/model-artifacts/registry.json | 485 +++++++++++++- .../src/api/routes/logs/events/query.rs | 15 +- .../src/inference/skippy/mod.rs | 67 +- .../src/logging/openai_lifecycle.rs | 5 + .../src/logging/request_metadata.rs | 5 + .../src/mesh/identity_persistence.rs | 2 +- crates/mesh-llm-host-runtime/src/mesh/mod.rs | 7 +- .../src/models/profile.rs | 1 + .../src/network/openai/ingress.rs | 137 +++- .../openai/ingress_tests/automatic_routing.rs | 315 +++++++++ .../src/network/openai/request_parse.rs | 122 +++- .../openai/request_parse/audio_multipart.rs | 180 +++++ .../src/network/openai/request_parse_tests.rs | 183 +++++ .../src/network/openai/response/models.rs | 5 + .../src/network/openai/routing_rank.rs | 130 ++++ .../src/network/openai/transport.rs | 12 +- .../network/openai/transport_tests/routing.rs | 49 ++ .../src/protocol/convert.rs | 70 ++ .../src/runtime/local.rs | 12 + .../src/runtime/local_split/loading.rs | 1 + .../src/runtime/local_split/test_support.rs | 2 + .../src/runtime/model_lifecycle.rs | 4 +- .../src/runtime/model_lifecycle/load.rs | 13 +- .../src/runtime/runtime_registry.rs | 24 +- .../src/runtime/startup_handles.rs | 21 +- .../src/runtime/tests/model_lifecycle.rs | 26 +- crates/mesh-llm-protocol/proto/node.proto | 10 + crates/mesh-llm-protocol/src/proto/node.rs | 40 ++ crates/mesh-llm-types/src/mesh/mod.rs | 23 +- crates/openai-frontend/Cargo.toml | 3 +- crates/openai-frontend/README.md | 22 +- crates/openai-frontend/src/audio.rs | 130 ++++ crates/openai-frontend/src/backend.rs | 55 ++ crates/openai-frontend/src/chat.rs | 4 +- crates/openai-frontend/src/embeddings.rs | 198 ++++++ .../openai-frontend/src/guardrails/compact.rs | 45 ++ crates/openai-frontend/src/guardrails/mod.rs | 45 ++ crates/openai-frontend/src/hooks.rs | 47 +- crates/openai-frontend/src/lib.rs | 11 + crates/openai-frontend/src/lifecycle.rs | 10 + crates/openai-frontend/src/rerank.rs | 92 +++ crates/openai-frontend/src/router.rs | 253 ++++++- crates/openai-frontend/src/router_tests.rs | 77 +++ .../src/router_tests/non_chat.rs | 223 +++++++ crates/skippy-ffi/build.rs | 76 ++- crates/skippy-ffi/src/abi.rs | 56 ++ crates/skippy-ffi/src/dynamic.rs | 29 +- crates/skippy-ffi/src/lib.rs | 154 +++-- crates/skippy-ffi/src/multimodal.rs | 42 ++ crates/skippy-ffi/src/static_bindings.rs | 82 ++- crates/skippy-ffi/src/tests.rs | 22 +- crates/skippy-quantize/src/compose_mtp.rs | 58 +- crates/skippy-runtime/src/lib.rs | 3 +- crates/skippy-runtime/src/media.rs | 247 +++++++ crates/skippy-runtime/src/native.rs | 137 +++- crates/skippy-runtime/src/session.rs | 67 ++ crates/skippy-server/README.md | 5 +- crates/skippy-server/src/embedded.rs | 19 +- crates/skippy-server/src/frontend/backend.rs | 227 ++++++- .../src/frontend/backend/non_chat.rs | 436 ++++++++++++ .../src/frontend/backend/tests.rs | 70 ++ .../src/frontend/generation_flow.rs | 1 + .../generation_flow/encoder_decoder.rs | 95 +++ .../generation_flow/text_generation.rs | 29 +- .../skippy-server/src/frontend/tests/mod.rs | 8 +- .../src/frontend/tests/multimodal.rs | 46 +- .../src/frontend/tests/non_chat.rs | 424 ++++++++++++ .../src/frontend/tests/support.rs | 43 ++ .../src/frontend/tests/tts_oracle.rs | 116 ++++ crates/skippy-server/src/runtime_state.rs | 99 ++- .../src/runtime_state/frame_operations.rs | 17 + docs/NON_CHAT_MODELS.md | 207 ++++++ docs/README.md | 1 + docs/SKIPPY.md | 11 +- docs/design/MULTI_MODAL.md | 18 +- just/ci.just | 5 + scripts/build-llama.sh | 28 +- scripts/check-skippy-workload-candidate.py | 33 + scripts/ci-openai-embeddings-smoke.py | 77 +++ scripts/ci-openai-workload-smoke.py | 272 ++++++++ scripts/ci-workload-monolithic-oracle.py | 245 +++++++ scripts/generate-ocr-oracle-fixture.py | 72 ++ scripts/generate-test-model-manifests.py | 33 +- scripts/llama-oracle-source.py | 80 +++ scripts/plan-family-battery.py | 123 +++- scripts/skippy-family-battery.sh | 261 ++++++-- scripts/skippy-ocr-asr-oracle.py | 216 ++++++ scripts/skippy-tts-oracle.py | 263 ++++++++ scripts/skippy-workload-certify.sh | 368 ++++++++++ .../test_check_skippy_workload_candidate.py | 67 ++ .../tests/test_llama_native_full_replay.py | 7 + scripts/tests/test_llama_oracle_source.py | 80 +++ .../test_llama_upstream_canary_contract.py | 71 +- scripts/tests/test_plan_family_battery.py | 217 +++++- scripts/tests/test_skippy_ocr_asr_oracle.py | 124 ++++ scripts/tests/test_skippy_static_link.py | 100 +++ scripts/tests/test_skippy_tts_oracle.py | 184 +++++ scripts/tests/test_skippy_workload_certify.py | 143 ++++ scripts/tests/test_static_abi_artifacts.py | 2 + .../test_verify_workload_oracle_evidence.py | 114 ++++ .../tests/test_workload_monolithic_oracle.py | 100 +++ scripts/verify-workload-oracle-evidence.py | 85 +++ scripts/workload_fixtures.py | 15 + scripts/write-workload-oracle-evidence.py | 78 +++ ...-non-chat-workload-ABI-and-execution.patch | 630 ++++++++++++++++++ ...ppy-configure-non-chat-model-loading.patch | 55 ++ ...or-vocab-suppress-tokens-in-sampling.patch | 348 ++++++++++ tools/xtask/data/console_print_allowlist.json | 14 +- website/src/docs/pages/skippy-api.md | 77 ++- 126 files changed, 10399 insertions(+), 436 deletions(-) create mode 100644 ci/llama-canary/fixtures/audio-smoke.wav create mode 100644 crates/mesh-llm-host-runtime/src/network/openai/request_parse/audio_multipart.rs create mode 100644 crates/openai-frontend/src/audio.rs create mode 100644 crates/openai-frontend/src/embeddings.rs create mode 100644 crates/openai-frontend/src/rerank.rs create mode 100644 crates/openai-frontend/src/router_tests/non_chat.rs create mode 100644 crates/skippy-server/src/frontend/backend/non_chat.rs create mode 100644 crates/skippy-server/src/frontend/generation_flow/encoder_decoder.rs create mode 100644 crates/skippy-server/src/frontend/tests/non_chat.rs create mode 100644 crates/skippy-server/src/frontend/tests/tts_oracle.rs create mode 100644 docs/NON_CHAT_MODELS.md create mode 100644 scripts/check-skippy-workload-candidate.py create mode 100755 scripts/ci-openai-embeddings-smoke.py create mode 100644 scripts/ci-openai-workload-smoke.py create mode 100644 scripts/ci-workload-monolithic-oracle.py create mode 100644 scripts/generate-ocr-oracle-fixture.py create mode 100644 scripts/llama-oracle-source.py create mode 100644 scripts/skippy-ocr-asr-oracle.py create mode 100644 scripts/skippy-tts-oracle.py create mode 100755 scripts/skippy-workload-certify.sh create mode 100644 scripts/tests/test_check_skippy_workload_candidate.py create mode 100644 scripts/tests/test_llama_oracle_source.py create mode 100644 scripts/tests/test_skippy_ocr_asr_oracle.py create mode 100644 scripts/tests/test_skippy_static_link.py create mode 100644 scripts/tests/test_skippy_tts_oracle.py create mode 100644 scripts/tests/test_skippy_workload_certify.py create mode 100644 scripts/tests/test_verify_workload_oracle_evidence.py create mode 100644 scripts/tests/test_workload_monolithic_oracle.py create mode 100644 scripts/verify-workload-oracle-evidence.py create mode 100644 scripts/workload_fixtures.py create mode 100644 scripts/write-workload-oracle-evidence.py create mode 100644 third_party/llama.cpp/patches/0024-skippy-define-non-chat-workload-ABI-and-execution.patch create mode 100644 third_party/llama.cpp/patches/0025-skippy-configure-non-chat-model-loading.patch create mode 100644 third_party/llama.cpp/patches/0026-skippy-honor-vocab-suppress-tokens-in-sampling.patch diff --git a/.agents/skills/manage-ci/references/current-inventory.md b/.agents/skills/manage-ci/references/current-inventory.md index 6505e3cf1a..d67a04996e 100644 --- a/.agents/skills/manage-ci/references/current-inventory.md +++ b/.agents/skills/manage-ci/references/current-inventory.md @@ -47,10 +47,16 @@ default-branch content only on the persistent self-hosted `family-certify` runner group (tools come from the runner image; no GitHub Actions model caching). Before native compilation, `scripts/plan-family-battery.py` validates the versioned JSON family policy, -the mandatory three-lane contract for every certified profile, and every exact -artifact revision/file in the immutable local cache. It reads only GGUF -metadata headers, requires each artifact to have at least one metadata-bearing -shard, and requires every shard that carries `*.block_count` and +the three core parity lanes for certified causal rows, one class-specific +smoke lane for each of the six registry-generated non-chat rows +(`embedding`, `rerank`, `encoder_decoder`, `ocr`, `speech_synthesis`, and +`speech_recognition`): respectively `embedding-smoke`, +`rerank-smoke`, `encoder-decoder-smoke`, `ocr-smoke`, +`speech-synthesis-smoke`, and `speech-recognition-smoke`. These lanes exercise +local full-model and HTTP behavior, without an independent equivalence oracle. It also +checks every exact artifact revision/file in the immutable local cache. It +reads only GGUF metadata headers, requires each artifact to have at least one +metadata-bearing shard, and requires every shard that carries `*.block_count` and `*.embedding_length` to equal the planned runtime range and activation width before compilation; Qwen4 experimental artifacts derive their wider boundary from `hyper_connection.count * embedding_length`. It emits @@ -77,9 +83,10 @@ Mamba). A changed pin selects `llama-bump`; a manual dispatch may set family battery. Before any certification starts, every selected GGUF is resolved directly by the immutable snapshot SHA checked into `ci/llama-canary/family-certified.json`. The runtime preflight records the -revisions, verifies all shard/tensor scans and declared runtime/MTP layer -counts/model bytes, disk -headroom and certification ports. Native MTP/NextN heads remain part of the +revisions and verifies all shard/tensor scans, declared runtime/MTP layer +counts/model bytes, and disk headroom. Full certification also probes the +certification port range; `--preflight-only` skips those socket probes and +records `port_range.checked=false`. Native MTP/NextN heads remain part of the single target model; the battery never reopens that model as a separate draft. Those rows require native draft sidebands in staged single-step and chain correctness, where each proposed token is verified against the target. The diff --git a/Cargo.lock b/Cargo.lock index 7b9a48bf76..c7a1d548ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -578,6 +578,7 @@ dependencies = [ "matchit", "memchr", "mime", + "multer", "percent-encoding", "pin-project-lite", "serde_core", @@ -4495,6 +4496,23 @@ dependencies = [ "pxfm", ] +[[package]] +name = "multer" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" +dependencies = [ + "bytes", + "encoding_rs", + "futures-util", + "http", + "httparse", + "memchr", + "mime", + "spin 0.9.9", + "version_check", +] + [[package]] name = "multimap" version = "0.10.1" @@ -5307,6 +5325,7 @@ version = "0.76.1" dependencies = [ "async-trait", "axum", + "base64 0.22.1", "futures-core", "futures-util", "http-body-util", diff --git a/ci/ci.md b/ci/ci.md index 4a8330d491..4b1992ee3f 100644 --- a/ci/ci.md +++ b/ci/ci.md @@ -26,7 +26,7 @@ and acceptance criteria are in `.omo/specs/pr-ci-optimization.md`. | `nightly-stability.yml` / `nightly-stability-run.yml` | daily schedule, dispatch / reusable | GitHub-hosted live-endpoint evidence. The general stability and KV tool-loop/prefix-reuse harnesses run independently, upload both evidence sets, and preserve either failure. The reusable workflow accepts no runner label. | | `nightly-kv-coverage.yml` | daily schedule, dispatch | Trusted-`main`, read-only, GitHub-hosted expansion of deterministic radix lease/eviction and blob-ownership state machines. Seed/step budgets and the exact source SHA are uploaded; no secrets or privileged runner are used. | | `agentic-replay-nightly.yml` | daily schedule, trusted-main dispatch | Opt-in coding-agent replay benchmark on the persistent macOS `micstudio` runner. The fixed `[self-hosted, X64, macOS, family-certify, agentic-replay]` selector is backed by a fail-closed `RUNNER_NAME=micstudio` check; both manual and scheduled execution check out trusted `main`. It resolves exact model and trajectory revisions from the pre-warmed Hugging Face cache, verifies their SHA-256 digests, uploads immutable evidence, publishes cohort-matched history, gates configured regressions, and may open a repair PR without executing pull-request content on the persistent runner. | -| `llama-upstream-canary.yml` | daily schedule, dispatch | Trusted default-branch llama.cpp bump certification on the self-hosted `family-certify` runner. It never runs as ordinary push or PR CI. Canary runs share a non-cancelling concurrency group, so a scheduled run queues behind active manual or scheduled work instead of discarding the candidate workspace. `scripts/plan-family-battery.py` validates the generated `ci/llama-canary/family-certified.json` policy (sourced from `ci/model-artifacts/registry.json`) and every file's exact immutable cache blob identity and byte size before native compilation. Each target/draft artifact must have at least one metadata-bearing GGUF shard; every shard that carries architecture dimensions must match the declared runtime range and activation width, including Qwen4's `hyper_connection.count * embedding_length` boundary. Optional `mmproj_artifact` rows pin a projector GGUF sidecar (exact blob identity, exempt from trunk-dimension checks), and each family that pins one runs an additional multimodal smoke lane after its core lanes: the real-projector + deterministic-image harness in `crates/skippy-server/src/frontend/tests/multimodal.rs` (local monolithic and split stages) via `SKIPPY_MM_*`, reconciled against the plan like every other lane. It emits deterministic bounded matrix shards and records the plan with evidence. The current single-runner workflow consumes one selected-family shard and builds the certification binaries once. Changed pins always run the complete `llama-bump` cohort. Before any lane starts, the battery verifies shard/tensor scans, declared runtime/MTP layer counts, model bytes, disk headroom and certification ports. Native MTP/NextN heads remain part of the single target model; the battery does not reopen the model as a separate draft. Those rows require native draft sidebands in staged single-step and chain correctness, where each proposed token is verified against the target. Every certified profile must retain strict `single-step`, `chain`, and `state-handoff` parity. Filtered correctness stages derive their exact resident tensor names from the native stage graph planner, including GGUFs with non-finite metadata values. Single-step and chain exercise the sole shipping raw-f32 activation wire and any mismatch is a hard failure. Planned families, sweep cuts, and multimodal smokes are reconciled exactly against executed lanes and recorded results. Declared per-model or model-size-derived startup deadlines, complete-certification wall-clock limits and typed lane outcomes are recorded, and immutable plans/model manifests/preflight evidence/certification logs upload even on failure or cancellation. Manual dispatch can force this certification when the upstream SHA is unchanged. Persistent-runner execution is always a read-only checkout of trusted `main`. Changed pins give one agent session the complete developer task: repair or regenerate the queue, address ABI fallout, and iterate through every canonical gate. The shared repair-and-test deadline is 450 minutes and the agent has no GitHub credentials. A zero exit from the coding process only yields control to the trusted wrapper; the wrapper runs the full candidate gates and returns current failure logs to the same session until those gates pass or the deadline expires. Existing certification and parity rows remain immutable. The agent may only correct `resources.estimated_model_bytes`, which the immutable GGUF scan rechecks, and append classification-only parity rows for source files missing from the manifest; those rows cannot add artifact or certification authority. Only a green repair pass is snapshotted as an unreachable local commit and uploaded as a thin candidate bundle. A separate self-hosted verification job and checkout download that bundle, materialize its commit in a fresh detached worktree, and independently run one ordered `prepare -> manifest-policy -> build -> certify` pass with a 240-minute budget and new native-build and family-evidence directories. Only the exact passing commit is exported as the certified bundle. A separate success-gated job on a fresh GitHub-hosted runner receives the repair token, validates the one-day bundle artifact, pushes a run-specific branch, and opens a normal exact-head PR as its final external mutation. Agent or verification failures retain logs and create no branch or PR. Successful publication leaves the canary green; changed pins are never pushed directly to `main`. Unchanged scheduled and forced certifications remain read-only and do not invoke the agent. Runner requires `HF_CACHE=/Users/lab/models/huggingface`, verifies its `hub` directory, exports `HF_HOME` and `HF_HUB_CACHE` from that root, and stays offline on the NFS-backed cache (`HF_HUB_OFFLINE=1`; no `flock`, so the runner never downloads). | +| `llama-upstream-canary.yml` | daily schedule, dispatch | Trusted default-branch llama.cpp bump certification on the self-hosted `family-certify` runner. It never runs as ordinary push or PR CI. Canary runs share a non-cancelling concurrency group, so a scheduled run queues behind active manual or scheduled work instead of discarding the candidate workspace. `scripts/plan-family-battery.py` validates the generated `ci/llama-canary/family-certified.json` policy (sourced from `ci/model-artifacts/registry.json`) and every file's exact immutable cache blob identity and byte size before native compilation. Each target/draft artifact must have at least one metadata-bearing GGUF shard; every shard that carries architecture dimensions must match the declared runtime range and activation width, including Qwen4's `hyper_connection.count * embedding_length` boundary. Optional `mmproj_artifact` rows pin a projector GGUF sidecar (exact blob identity, exempt from trunk-dimension checks). Causal-generation families with a projector run an additional multimodal smoke lane after their core lanes: the real-projector + deterministic-image harness in `crates/skippy-server/src/frontend/tests/multimodal.rs` (local monolithic and split stages) via `SKIPPY_MM_*`. Non-chat projector rows run their class-specific workload lanes instead. It emits deterministic bounded matrix shards and records the plan with evidence. The current single-runner workflow consumes one selected-family shard and builds the certification binaries once. Changed pins always run the complete `llama-bump` cohort. Before any lane starts, the battery verifies shard/tensor scans, declared runtime/MTP layer counts, model bytes and disk headroom. Full certification also probes the certification port range; `--preflight-only` skips socket probes and records `port_range.checked=false`. Native MTP/NextN heads remain part of the single target model; the battery does not reopen the model as a separate draft. Those rows require native draft sidebands in staged single-step and chain correctness, where each proposed token is verified against the target. Certified causal-generation profiles retain strict `single-step`, `chain`, and `state-handoff` parity. The six manual-full non-chat rows run class-specific full-model smoke and independent local-monolithic oracle lanes: embedding, rerank, encoder-decoder, OCR, speech synthesis, and speech recognition. The smoke lane alone does not establish oracle equivalence. Filtered correctness stages derive their exact resident tensor names from the native stage graph planner, including GGUFs with non-finite metadata values. Single-step and chain exercise the sole shipping raw-f32 activation wire and any mismatch is a hard failure. Planned families, sweep cuts, class-specific workload lanes, and multimodal smokes are reconciled exactly against executed lanes and recorded results. Declared per-model or model-size-derived startup deadlines, complete-certification wall-clock limits and typed lane outcomes are recorded, and immutable plans/model manifests/preflight evidence/certification logs upload even on failure or cancellation. Manual dispatch can force this certification when the upstream SHA is unchanged. Persistent-runner execution is always a read-only checkout of trusted `main`. Changed pins give one agent session the complete developer task: repair or regenerate the queue, address ABI fallout, and iterate through every canonical gate. The shared repair-and-test deadline is 450 minutes and the agent has no GitHub credentials. A zero exit from the coding process only yields control to the trusted wrapper; the wrapper runs the full candidate gates and returns current failure logs to the same session until those gates pass or the deadline expires. Existing certification and parity rows remain immutable. The agent may only correct `resources.estimated_model_bytes`, which the immutable GGUF scan rechecks, and append classification-only parity rows for source files missing from the manifest; those rows cannot add artifact or certification authority. Only a green repair pass is snapshotted as an unreachable local commit and uploaded as a thin candidate bundle. A separate self-hosted verification job and checkout download that bundle, materialize its commit in a fresh detached worktree, and independently run one ordered `prepare -> manifest-policy -> build -> certify` pass with a 240-minute budget and new native-build and family-evidence directories. Only the exact passing commit is exported as the certified bundle. A separate success-gated job on a fresh GitHub-hosted runner receives the repair token, validates the one-day bundle artifact, pushes a run-specific branch, and opens a normal exact-head PR as its final external mutation. Agent or verification failures retain logs and create no branch or PR. Successful publication leaves the canary green; changed pins are never pushed directly to `main`. Unchanged scheduled and forced certifications remain read-only and do not invoke the agent. Runner requires `HF_CACHE=/Users/lab/models/huggingface`, verifies its `hub` directory, exports `HF_HOME` and `HF_HUB_CACHE` from that root, and stays offline on the NFS-backed cache (`HF_HUB_OFFLINE=1`; no `flock`, so the runner never downloads). | The changed-pin canary wrapper owns the target-pin transition: it writes the sole upstream selector, `third_party/llama.cpp/upstream.txt`, before the agent diff --git a/ci/llama-canary/family-certified.json b/ci/llama-canary/family-certified.json index f50712c5b1..a12211aa31 100644 --- a/ci/llama-canary/family-certified.json +++ b/ci/llama-canary/family-certified.json @@ -16,6 +16,16 @@ "status": "provisional", "oracle": "none", "required_lanes": ["graph-parse", "tensor-ownership", "stage-load"] + }, + "workload-smoke": { + "status": "provisional", + "oracle": "none", + "required_lanes": ["class-specific-smoke"] + }, + "workload-oracle": { + "status": "certified", + "oracle": "local-monolithic", + "required_lanes": ["class-specific-smoke", "class-specific-oracle"] } }, "cadences": ["llama-bump", "manual-full", "nightly", "rotating"] @@ -23,6 +33,7 @@ "models": [ { "family": "qwen3-dense", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full", "nightly"], "artifact": {"repo": "Qwen/Qwen3-0.6B-GGUF", "revision": "23749fefcc72300e3a2ad315e1317431b06b590a", "files": ["Qwen3-0.6B-Q8_0.gguf"], "file_integrity": {"Qwen3-0.6B-Q8_0.gguf": {"size_bytes": 639446688, "blob_id": "9465e63a22add5354d9bb4b99e90117043c7124007664907259bd16d043bb031"}}, "selector": "Q8_0"}, @@ -32,6 +43,7 @@ }, { "family": "llama", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "bartowski/Llama-3.2-1B-Instruct-GGUF", "revision": "067b946cf014b7c697f3654f621d577a3e3afd1c", "files": ["Llama-3.2-1B-Instruct-Q4_K_M.gguf"], "file_integrity": {"Llama-3.2-1B-Instruct-Q4_K_M.gguf": {"size_bytes": 807694464, "blob_id": "6f85a640a97cf2bf5b8e764087b1e83da0fdb51d7c9fab7d0fece9385611df83"}}, "selector": "Q4_K_M"}, @@ -41,6 +53,7 @@ }, { "family": "glm47-flash", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "unsloth/GLM-4.7-Flash-GGUF", "revision": "0d32489ecb9db6d2a4fc93bd27ef01519f95474d", "files": ["GLM-4.7-Flash-Q4_K_M.gguf"], "file_integrity": {"GLM-4.7-Flash-Q4_K_M.gguf": {"size_bytes": 18312339808, "blob_id": "29837ed2c0fc5f51981adf8ac8083fcf80743c598381f13e9f06cbad0498b174"}}, "selector": "Q4_K_M"}, @@ -50,6 +63,7 @@ }, { "family": "glm4", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "bartowski/glm-4-9b-chat-GGUF", "revision": "74a33eafa149550a60bff0fdafcdd34caa6df9f0", "files": ["glm-4-9b-chat-Q4_K_M.gguf"], "file_integrity": {"glm-4-9b-chat-Q4_K_M.gguf": {"size_bytes": 6250926848, "blob_id": "aa6cb8f5ef0a70399bdbf92eef566c26c6017ee0b131a424b65b5c757f1c81a2"}}, "selector": "Q4_K_M"}, @@ -59,6 +73,7 @@ }, { "family": "glm45-air", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "unsloth/GLM-4.5-Air-GGUF", "revision": "506d64aa8c5cfe9dbbf00bc7a15739438f83204d", "files": ["Q4_K_M/GLM-4.5-Air-Q4_K_M-00001-of-00002.gguf", "Q4_K_M/GLM-4.5-Air-Q4_K_M-00002-of-00002.gguf"], "file_integrity": {"Q4_K_M/GLM-4.5-Air-Q4_K_M-00001-of-00002.gguf": {"size_bytes": 50000746752, "blob_id": "3e7e5d25a6db33b7c90f8c5203d6f490d6c0a4f04a46f228af0e33727db5df5e"}, "Q4_K_M/GLM-4.5-Air-Q4_K_M-00002-of-00002.gguf": {"size_bytes": 22975001632, "blob_id": "ac6e50523d45c53faf3bf72d8fe109f3dd54bc6abe98a7773a3636112ead82a6"}}, "selector": "Q4_K_M"}, @@ -68,6 +83,7 @@ }, { "family": "deepseek2", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "bartowski/DeepSeek-Coder-V2-Lite-Instruct-GGUF", "revision": "8f248fa2072348f77a8bc37754e470de1f61866e", "files": ["DeepSeek-Coder-V2-Lite-Instruct-Q4_K_M.gguf"], "file_integrity": {"DeepSeek-Coder-V2-Lite-Instruct-Q4_K_M.gguf": {"size_bytes": 10364416768, "blob_id": "603bd3f8a0281d16571da7c08bd661ee17ff0d1be6fcbd1b42242da257ef0bb8"}}, "selector": "Q4_K_M"}, @@ -77,6 +93,7 @@ }, { "family": "gpt-oss", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "ggml-org/gpt-oss-20b-GGUF", "revision": "ef9b12f2ff56c69cf32153a02784e7a3c88bf524", "files": ["gpt-oss-20b-MXFP4.gguf"], "file_integrity": {"gpt-oss-20b-MXFP4.gguf": {"size_bytes": 12109566624, "blob_id": "27cd6c432c7672cb812a92f611cf3ba7bbc35928262bb1e1253ff4ee6ae35901"}}, "selector": "MXFP4"}, @@ -86,6 +103,7 @@ }, { "family": "qwen3-moe", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "unsloth/Qwen3-30B-A3B-Instruct-2507-GGUF", "revision": "eea7b2be5805a5f151f8847ede8e5f9a9284bf77", "files": ["Qwen3-30B-A3B-Instruct-2507-Q4_K_M.gguf"], "file_integrity": {"Qwen3-30B-A3B-Instruct-2507-Q4_K_M.gguf": {"size_bytes": 18556686752, "blob_id": "6c997b8af17debdfb01d890214400ccbab00db6acc0ba8da5de1cc906c4774d0"}}, "selector": "Q4_K_M"}, @@ -95,6 +113,7 @@ }, { "family": "qwen2-moe", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "RichardErkhov/Qwen_-_Qwen1.5-MoE-A2.7B-Chat-gguf", "revision": "947580e7b3904cd4081810d0efec4520a75609c7", "files": ["Qwen1.5-MoE-A2.7B-Chat.Q4_K_M.gguf"], "file_integrity": {"Qwen1.5-MoE-A2.7B-Chat.Q4_K_M.gguf": {"size_bytes": 9496236768, "blob_id": "3dcfb1032781788bf373edb49b888fcac5b6093ea0fe0edf5e23a977390b3eea"}}, "selector": "Q4_K_M"}, @@ -104,6 +123,7 @@ }, { "family": "falcon-h1", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full", "nightly"], "artifact": {"repo": "tiiuae/Falcon-H1-1.5B-Instruct-GGUF", "revision": "0d3a6cfe25fb4eeab0153fb8623aac5b69d6bd0a", "files": ["Falcon-H1-1.5B-Instruct-Q4_K_M.gguf"], "file_integrity": {"Falcon-H1-1.5B-Instruct-Q4_K_M.gguf": {"size_bytes": 944786656, "blob_id": "8b51aa2aa34a0373fd0cd64c02eb91d1bc1da681c09e955ad769d4a9b2d8385f"}}, "selector": "Q4_K_M"}, @@ -113,6 +133,7 @@ }, { "family": "jamba2", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "bartowski/ai21labs_AI21-Jamba2-3B-GGUF", "revision": "02d70acd708332ec4e78e9ceefe116851a307411", "files": ["ai21labs_AI21-Jamba2-3B-Q4_K_M.gguf"], "file_integrity": {"ai21labs_AI21-Jamba2-3B-Q4_K_M.gguf": {"size_bytes": 1932696448, "blob_id": "fa0876fa152f38689cefd6d498907d6255cf515780d65b16346d3190df6a0794"}}, "selector": "Q4_K_M"}, @@ -122,6 +143,7 @@ }, { "family": "kimi-linear", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "bartowski/moonshotai_Kimi-Linear-48B-A3B-Instruct-GGUF", "revision": "228dbe476e5a02091624a19068f4c962caa8a1c5", "files": ["moonshotai_Kimi-Linear-48B-A3B-Instruct-IQ2_XS.gguf"], "file_integrity": {"moonshotai_Kimi-Linear-48B-A3B-Instruct-IQ2_XS.gguf": {"size_bytes": 13897237408, "blob_id": "883be5561005ed6558c3f9d58d33bd9df818ea41c18086a49769047278953c0e"}}, "selector": "IQ2_XS"}, @@ -131,6 +153,7 @@ }, { "family": "qwen3-next", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full", "nightly"], "artifact": {"repo": "bartowski/Qwen_Qwen3-Next-80B-A3B-Thinking-GGUF", "revision": "c36ac5f15be18742fcd09baa61d48dfc016aae79", "files": ["Qwen_Qwen3-Next-80B-A3B-Thinking-IQ2_XS.gguf"], "file_integrity": {"Qwen_Qwen3-Next-80B-A3B-Thinking-IQ2_XS.gguf": {"size_bytes": 22217422720, "blob_id": "ece0ffe31891d1a0f1993565db4a977ae6d862d0ae2e45fc7282ff0a06cf5d5d"}}, "selector": "IQ2_XS"}, @@ -140,6 +163,7 @@ }, { "family": "qwen4exp", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "unsloth/Qwen3.8-Flash-Next-GGUF", "revision": "c8b5954a88c2775c546b92593eda40ea041d3176", "files": ["UD-IQ1_S/Qwen3.8-Flash-Next-UD-IQ1_S-00001-of-00003.gguf", "UD-IQ1_S/Qwen3.8-Flash-Next-UD-IQ1_S-00002-of-00003.gguf", "UD-IQ1_S/Qwen3.8-Flash-Next-UD-IQ1_S-00003-of-00003.gguf"], "file_integrity": {"UD-IQ1_S/Qwen3.8-Flash-Next-UD-IQ1_S-00001-of-00003.gguf": {"size_bytes": 10946624, "blob_id": "88a1420825a9304063e882ada29d438263617f51ac8923d438d927496693bafd"}, "UD-IQ1_S/Qwen3.8-Flash-Next-UD-IQ1_S-00002-of-00003.gguf": {"size_bytes": 49990818368, "blob_id": "3a62e35bbf9add4733bd1438ebd3a67649d5edd6cb0e72bb78e33c913992b2b6"}, "UD-IQ1_S/Qwen3.8-Flash-Next-UD-IQ1_S-00003-of-00003.gguf": {"size_bytes": 22544696352, "blob_id": "0e25ceaeb89b8a80aa973c6c0c7448943682f7408c2855b2ebd016b7643a861a"}}, "selector": "UD-IQ1_S"}, @@ -149,6 +173,7 @@ }, { "family": "nemotron", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF", "revision": "f2d3fe3694501008786e81e5f20360cbf715496a", "files": ["NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q4_K_M.gguf"], "file_integrity": {"NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q4_K_M.gguf": {"size_bytes": 25266255936, "blob_id": "edcb5d4650796ed2fb412498de6f83b585862312c747ddb74f0ea04b22206181"}}, "selector": "Q4_K_M"}, @@ -158,6 +183,7 @@ }, { "family": "mamba", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full", "nightly"], "artifact": {"repo": "Felladrin/gguf-mamba-130m-hf", "revision": "83ef2222e1a0437d26bda213537afc195a53b3ad", "files": ["mamba-130m-hf.Q4_K_M.gguf"], "file_integrity": {"mamba-130m-hf.Q4_K_M.gguf": {"size_bytes": 103576544, "blob_id": "2784131bc0546f802bcb720cbc77d26c174c85514400cc645507146ad95f07da"}}, "selector": "Q4_K_M"}, @@ -167,6 +193,7 @@ }, { "family": "mamba2", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "bartowski/falcon-mamba-7b-GGUF", "revision": "1236a076ebddbcfcbd6ba0fe310d473d20527f11", "files": ["falcon-mamba-7b-Q4_K_M.gguf"], "file_integrity": {"falcon-mamba-7b-Q4_K_M.gguf": {"size_bytes": 4204230656, "blob_id": "ef8ba78a1fd0b0fd1d1c966acc5f4c043b12fbbd7044f4d9aad82e6b7a51e0a7"}}, "selector": "Q4_K_M"}, @@ -176,6 +203,7 @@ }, { "family": "rwkv6", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "latestissue/rwkv-6-finch-1b6-gguf", "revision": "9f4420bafb8fe463599b9983d81e12e1868c3746", "files": ["rwkv-6-finch-1b6-Q4_0.gguf"], "file_integrity": {"rwkv-6-finch-1b6-Q4_0.gguf": {"size_bytes": 992738848, "blob_id": "7b345728c3792f0168a615bd529a319d63f8becb760090330ce583a24f55e46b"}}, "selector": "Q4_0"}, @@ -185,6 +213,7 @@ }, { "family": "rwkv7", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "Mungert/RWKV7-Goose-World3-2.9B-HF-GGUF", "revision": "e85c69858772a4d6b481d99e1e49910dad48e123", "files": ["RWKV7-Goose-World3-2.9B-HF-f16-q4_k.gguf"], "file_integrity": {"RWKV7-Goose-World3-2.9B-HF-f16-q4_k.gguf": {"size_bytes": 2314884576, "blob_id": "bfcb5b239cf99ac4a590d7163266bbe3bb14f41d221ad6f7a2bf602b610211b0"}}, "selector": "Q4_K"}, @@ -194,6 +223,7 @@ }, { "family": "granite-hybrid", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "ibm-granite/granite-4.0-h-350m-GGUF", "revision": "a864f823cce6e6048b5752e2816fe7a23987d790", "files": ["granite-4.0-h-350m-Q4_K_M.gguf"], "file_integrity": {"granite-4.0-h-350m-Q4_K_M.gguf": {"size_bytes": 222662560, "blob_id": "0a8d6a7373602fadfba274a640ba784b86cc6847f1c67f1b0a90fa2ec266b7fb"}}, "selector": "Q4_K_M"}, @@ -203,6 +233,7 @@ }, { "family": "qwen35", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "unsloth/Qwen3.5-0.8B-GGUF", "revision": "6ab461498e2023f6e3c1baea90a8f0fe38ab64d0", "files": ["Qwen3.5-0.8B-Q4_K_M.gguf"], "file_integrity": {"Qwen3.5-0.8B-Q4_K_M.gguf": {"size_bytes": 532517120, "blob_id": "bd258782e35f7f458f8aced1adc053e6e92e89bc735ba3be89d38a06121dc517"}}, "selector": "Q4_K_M"}, @@ -212,6 +243,7 @@ }, { "family": "granite-dense", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "bartowski/ibm-granite_granite-3.2-2b-instruct-GGUF", "revision": "9be2c106c8c073f5a140ec03c8c7a6a6e72d097b", "files": ["ibm-granite_granite-3.2-2b-instruct-Q4_K_M.gguf"], "file_integrity": {"ibm-granite_granite-3.2-2b-instruct-Q4_K_M.gguf": {"size_bytes": 1545296512, "blob_id": "e1b915b0849becf4fdda188dee7b09cbebbfabd71c6f3f2b75dd3eca0a8fded1"}}, "selector": "Q4_K_M"}, @@ -221,6 +253,7 @@ }, { "family": "lfm2", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "LiquidAI/LFM2-350M-GGUF", "revision": "8fdc9d526b7ed346b19257551b05816c7912ecc2", "files": ["LFM2-350M-Q4_K_M.gguf"], "file_integrity": {"LFM2-350M-Q4_K_M.gguf": {"size_bytes": 229309376, "blob_id": "a4d000c7064bd3b2e42c6845836286a899a4e79cf1791da1a6797b58d575957d"}}, "selector": "Q4_K_M"}, @@ -230,6 +263,7 @@ }, { "family": "lfm2-vl", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "ggml-org/LFM2-VL-450M-GGUF", "revision": "0ff2ce5a6710f9bf0a6de87ec91832b43fa0ac05", "files": ["LFM2-VL-450M-Q8_0.gguf"], "file_integrity": {"LFM2-VL-450M-Q8_0.gguf": {"size_bytes": 379215264, "blob_id": "e97704a0cf0a1d00ca604b4c672c82f4234318dba9d43a7f7a4c0d2df6747844"}}, "selector": "Q8_0"}, @@ -240,6 +274,7 @@ }, { "family": "laguna", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "poolside/Laguna-XS-2.1-GGUF", "revision": "1a37c0a5fb8c7a18e6106decb6be6327d1b63fa6", "files": ["Laguna-XS-2.1-Q4_K_M.gguf"], "file_integrity": {"Laguna-XS-2.1-Q4_K_M.gguf": {"size_bytes": 20274300032, "blob_id": "1ac7079101fca5a6df8c5a7523a3c30ea7d1c0e4b1258090e7d6d4039287f6cb"}}, "selector": "Q4_K_M"}, @@ -249,6 +284,7 @@ }, { "family": "qwen2-vl", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "bartowski/Qwen2-VL-2B-Instruct-GGUF", "revision": "2160e265f926e84e84dbb1c73e623b5324b23707", "files": ["Qwen2-VL-2B-Instruct-Q4_K_M.gguf"], "file_integrity": {"Qwen2-VL-2B-Instruct-Q4_K_M.gguf": {"size_bytes": 986047232, "blob_id": "4ef095263343fc1237e8ca879790bb262bcf209f082e0a9bfce219b7ece55e8b"}}, "selector": "Q4_K_M"}, @@ -259,6 +295,7 @@ }, { "family": "qwen3-vl", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "unsloth/Qwen3-VL-8B-Instruct-GGUF", "revision": "b93a7ee713758252c555be4210c00540df954dc2", "files": ["Qwen3-VL-8B-Instruct-Q4_K_M.gguf"], "file_integrity": {"Qwen3-VL-8B-Instruct-Q4_K_M.gguf": {"size_bytes": 5027785568, "blob_id": "108e7ff92b78eefd3db4741885104acba514255c11b617d3c7b197a5f46efe89"}}, "selector": "Q4_K_M"}, @@ -269,6 +306,7 @@ }, { "family": "gemma2", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "google/gemma-2-2b-it-GGUF", "revision": "67803c710cb0c89bae2047a5f50a2481c9339d24", "files": ["2b_it_v2.gguf"], "file_integrity": {"2b_it_v2.gguf": {"size_bytes": 10463413632, "blob_id": "8fc1a799613b873edd085ffcf33fd710077daf03833ead29e136cca9b528ecb6"}}, "selector": "2b_it_v2"}, @@ -278,6 +316,7 @@ }, { "family": "phi3", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "microsoft/Phi-3-mini-4k-instruct-gguf", "revision": "a64113399c2f6b8ad3e11c394733a2ddadaa7f33", "files": ["Phi-3-mini-4k-instruct-q4.gguf"], "file_integrity": {"Phi-3-mini-4k-instruct-q4.gguf": {"size_bytes": 2393231072, "blob_id": "8a83c7fb9049a9b2e92266fa7ad04933bb53aa1e85136b7b30f1b8000ff2edef"}}, "selector": "q4"}, @@ -287,6 +326,7 @@ }, { "family": "phi4", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "unsloth/phi-4-GGUF", "revision": "5110b7771e8166d5530e73346a15aea096a8cb99", "files": ["phi-4-Q4_K_M.gguf"], "file_integrity": {"phi-4-Q4_K_M.gguf": {"size_bytes": 8890306112, "blob_id": "01e1f25b3e6931054c6c2227b06f4969828434eebc299e8e171f55dab6814485"}}, "selector": "Q4_K_M"}, @@ -296,6 +336,7 @@ }, { "family": "falcon-dense", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "maddes8cht/tiiuae-falcon-7b-instruct-gguf", "revision": "56d71fbb3ea040a1d35459013bd2688613075668", "files": ["tiiuae-falcon-7b-instruct-Q4_K_M.gguf"], "file_integrity": {"tiiuae-falcon-7b-instruct-Q4_K_M.gguf": {"size_bytes": 4975385792, "blob_id": "6f6c886ed07d4f6133a4dd7f8b799764ed719c50973d96816b2ac5ff3ada9913"}}, "selector": "Q4_K_M"}, @@ -305,6 +346,7 @@ }, { "family": "internlm2", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "Felladrin/gguf-internlm2-chat-1_8b", "revision": "cdd579415ed57fca8af5c5e09b35ee5124449876", "files": ["internlm2-chat-1_8b.F16.gguf"], "file_integrity": {"internlm2-chat-1_8b.F16.gguf": {"size_bytes": 3780559520, "blob_id": "377e44d5967ea2a51570fbc28448dd8693d0139c73b13b9e773fad783f7bd957"}}, "selector": "F16"}, @@ -314,6 +356,7 @@ }, { "family": "mistral-small", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "unsloth/Mistral-Small-3.2-24B-Instruct-2506-GGUF", "revision": "b750ec2299225e492f1bd27cab88a0a595fa848f", "files": ["Mistral-Small-3.2-24B-Instruct-2506-Q4_K_M.gguf"], "file_integrity": {"Mistral-Small-3.2-24B-Instruct-2506-Q4_K_M.gguf": {"size_bytes": 14333922848, "blob_id": "a3cc56310807ed0d145eaf9f018ccda9ae7ad8edb41ec870aa2454b0d4700b3c"}}, "selector": "Q4_K_M"}, @@ -323,6 +366,7 @@ }, { "family": "cohere2", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "bartowski/c4ai-command-r7b-12-2024-GGUF", "revision": "bfc7a934c45cb839d84c8ca01d87f1cfa51aaa3f", "files": ["c4ai-command-r7b-12-2024-Q4_K_M.gguf"], "file_integrity": {"c4ai-command-r7b-12-2024-Q4_K_M.gguf": {"size_bytes": 5057009792, "blob_id": "0653c1915f7e4819a980deea8e9ba86b0a78cbb283712785cbf091c3ffbe3b13"}}, "selector": "Q4_K_M"}, @@ -332,6 +376,7 @@ }, { "family": "smollm3", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "ggml-org/SmolLM3-3B-GGUF", "revision": "4965cb60b150737b68a0408c36aeefb65078f894", "files": ["SmolLM3-Q4_K_M.gguf"], "file_integrity": {"SmolLM3-Q4_K_M.gguf": {"size_bytes": 1915305312, "blob_id": "8334b850b7bd46238c16b0c550df2138f0889bf433809008cc17a8b05761863e"}}, "selector": "Q4_K_M"}, @@ -341,6 +386,7 @@ }, { "family": "qwen2", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "Qwen/Qwen2-0.5B-Instruct-GGUF", "revision": "198f08841147e5196a6a69bd0053690fb1fd3857", "files": ["qwen2-0_5b-instruct-q4_k_m.gguf"], "file_integrity": {"qwen2-0_5b-instruct-q4_k_m.gguf": {"size_bytes": 397805248, "blob_id": "f0a42bb979ca62b5e61f3bf924ab4b6a40aa091825ee7dcb4039949980ab81a8"}}, "selector": "Q4_K_M"}, @@ -350,6 +396,7 @@ }, { "family": "bloom", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "afrideva/bloom-560m-GGUF", "revision": "fe8ad7e97957783ea24f6960f73e28e709aa6e96", "files": ["bloom-560m.q4_k_m.gguf"], "file_integrity": {"bloom-560m.q4_k_m.gguf": {"size_bytes": 561444928, "blob_id": "383bc73de0c2e96704301b27117e7fd209255eda0474198dafdf4dd61ad7bf3e"}}, "selector": "Q4_K_M"}, @@ -359,6 +406,7 @@ }, { "family": "gemma", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "lmstudio-ai/gemma-2b-it-GGUF", "revision": "a0b140bfb922a743f89dd0682a24a17516071ab9", "files": ["gemma-2b-it-q4_k_m.gguf"], "file_integrity": {"gemma-2b-it-q4_k_m.gguf": {"size_bytes": 1495245728, "blob_id": "144b3eb4fb500556034a29172cfa2b17f28cfce7c7698ac3467600adc510efcd"}}, "selector": "Q4_K_M"}, @@ -368,6 +416,7 @@ }, { "family": "gemma3", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "ggml-org/gemma-3-1b-it-GGUF", "revision": "f9c28bcd85737ffc5aef028638d3341d49869c27", "files": ["gemma-3-1b-it-Q4_K_M.gguf"], "file_integrity": {"gemma-3-1b-it-Q4_K_M.gguf": {"size_bytes": 806058240, "blob_id": "8ccc5cd1f1b3602548715ae25a66ed73fd5dc68a210412eea643eb20eb75a135"}}, "selector": "Q4_K_M"}, @@ -377,6 +426,7 @@ }, { "family": "phi2", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "TheBloke/phi-2-GGUF", "revision": "5a454d977c6438bb9fb2df233c8ca70f21c87420", "files": ["phi-2.Q4_K_M.gguf"], "file_integrity": {"phi-2.Q4_K_M.gguf": {"size_bytes": 1789239136, "blob_id": "324356668fa5ba9f4135de348447bb2bbe2467eaa1b8fcfb53719de62fbd2499"}}, "selector": "Q4_K_M"}, @@ -386,6 +436,7 @@ }, { "family": "starcoder2", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "second-state/StarCoder2-3B-GGUF", "revision": "7fca3e2da2ce31df411461e2cb9cae2d2b492f35", "files": ["starcoder2-3b-Q4_K_M.gguf"], "file_integrity": {"starcoder2-3b-Q4_K_M.gguf": {"size_bytes": 1848976448, "blob_id": "d8fb39287a463549b80d97473b0a7595c3a5a6da3ae2604ca33906a1a43f7175"}}, "selector": "Q4_K_M"}, @@ -395,6 +446,7 @@ }, { "family": "stablelm", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "second-state/stablelm-2-zephyr-1.6b-GGUF", "revision": "6da77f49f48f36f1a01753ca27ba531bde13b861", "files": ["stablelm-2-zephyr-1_6b-Q4_K_M.gguf"], "file_integrity": {"stablelm-2-zephyr-1_6b-Q4_K_M.gguf": {"size_bytes": 1031442432, "blob_id": "4224a8a08417e34d5a224efb1291491c4f5762b46aa012021adaf0a0578848e2"}}, "selector": "Q4_K_M"}, @@ -404,6 +456,7 @@ }, { "family": "olmo2", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "allenai/OLMo-2-0425-1B-Instruct-GGUF", "revision": "62f8c199538474c3e33ed5d7e0580abd66686a27", "files": ["OLMo-2-0425-1B-Instruct-Q4_K_M.gguf"], "file_integrity": {"OLMo-2-0425-1B-Instruct-Q4_K_M.gguf": {"size_bytes": 935515296, "blob_id": "abd8187934a438fbf7cfff0a1de5b9d2793ce913f158794df1951dcba6c93cc6"}}, "selector": "Q4_K_M"}, @@ -413,6 +466,7 @@ }, { "family": "exaone4", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "LGAI-EXAONE/EXAONE-4.0-1.2B-GGUF", "revision": "162446400ea4596377a3ce1d3ddffa32971af0a6", "files": ["EXAONE-4.0-1.2B-Q4_K_M.gguf"], "file_integrity": {"EXAONE-4.0-1.2B-Q4_K_M.gguf": {"size_bytes": 812437792, "blob_id": "7b5e753540183ae4d56e6febd9b48cdd944de53386e6faa8f51c8f98cb2b47df"}}, "selector": "Q4_K_M"}, @@ -422,6 +476,7 @@ }, { "family": "minicpm3", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "openbmb/MiniCPM3-4B-GGUF", "revision": "816dc79b35f92827e0d2d87aacea3567e49661a8", "files": ["minicpm3-4b-q4_k_m.gguf"], "file_integrity": {"minicpm3-4b-q4_k_m.gguf": {"size_bytes": 2469791584, "blob_id": "64913247e927414ecf47fd3e9ea8e3f0c9acae293f583dfa7e24b8872e20fa4c"}}, "selector": "Q4_K_M"}, @@ -431,6 +486,7 @@ }, { "family": "nemotron-expansion", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "bartowski/Nemotron-Mini-4B-Instruct-GGUF", "revision": "fb49cde090c86092d89905bea2ffc41c23c2615e", "files": ["Nemotron-Mini-4B-Instruct-Q4_K_M.gguf"], "file_integrity": {"Nemotron-Mini-4B-Instruct-Q4_K_M.gguf": {"size_bytes": 2697387072, "blob_id": "2bf02846dbd45e9580b9338b90505fb54e00d185713d4b7f06699e2d5d298e48"}}, "selector": "Q4_K_M"}, @@ -440,6 +496,7 @@ }, { "family": "arcee", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "arcee-ai/AFM-4.5B-GGUF", "revision": "741fded55739927f94be7934a174c2bcfc751969", "files": ["AFM-4.5B-Q4_K_M.gguf"], "file_integrity": {"AFM-4.5B-Q4_K_M.gguf": {"size_bytes": 2916301824, "blob_id": "f05516b323f581bebae1af2cbf900d83a2569b0a60c54366daf4a9c15ae30d4f"}}, "selector": "Q4_K_M"}, @@ -449,6 +506,7 @@ }, { "family": "mpt", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "maddes8cht/mosaicml-mpt-7b-instruct-gguf", "revision": "939bf1d3b1fcab272e542599bfaf74dcd04a9931", "files": ["mosaicml-mpt-7b-instruct-Q4_K_M.gguf"], "file_integrity": {"mosaicml-mpt-7b-instruct-Q4_K_M.gguf": {"size_bytes": 4390524128, "blob_id": "bfc1cf1333d60dd8f4c9ea97652ebdb984501c389d22cebc02b564f45c68d628"}}, "selector": "Q4_K_M"}, @@ -458,6 +516,7 @@ }, { "family": "apertus", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "bartowski/swiss-ai_Apertus-8B-Instruct-2509-GGUF", "revision": "d2ffbcc459d0a1bccb2097aca8bb825974f7678c", "files": ["swiss-ai_Apertus-8B-Instruct-2509-Q4_K_M.gguf"], "file_integrity": {"swiss-ai_Apertus-8B-Instruct-2509-Q4_K_M.gguf": {"size_bytes": 5057885568, "blob_id": "5901007f15aec9aabeea401cbe65559671166fc851bbdf8c26f56c2b61e9cd1d"}}, "selector": "Q4_K_M"}, @@ -467,6 +526,7 @@ }, { "family": "glm4-expansion", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "unsloth/GLM-4-9B-0414-GGUF", "revision": "40a17a0c8f24664ddd851dafd03935553c25a57e", "files": ["GLM-4-9B-0414-Q4_K_M.gguf"], "file_integrity": {"GLM-4-9B-0414-Q4_K_M.gguf": {"size_bytes": 6166574944, "blob_id": "8027e1089273e8817b2df0d91c9aa17c5ea467246dcdacac34989f8919fe6540"}}, "selector": "Q4_K_M"}, @@ -476,6 +536,7 @@ }, { "family": "afmoe", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "MaziyarPanahi/Trinity-Mini-GGUF", "revision": "3a7d7a64abeabff7302851a2bc98508b36d0b8f7", "files": ["Trinity-Mini.Q4_K_M.gguf"], "file_integrity": {"Trinity-Mini.Q4_K_M.gguf": {"size_bytes": 15823053440, "blob_id": "2c43474de9017d68331455cbf3852c1d365f7a5f9d3b050418e024ba3374a19e"}}, "selector": "Q4_K_M"}, @@ -485,6 +546,7 @@ }, { "family": "arwkv7", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "mradermacher/ARWKV-7B-Preview-0.1-NoG-32B-i1-GGUF", "revision": "301a9554c8271bca175a78f599b20e49bce383fa", "files": ["ARWKV-7B-Preview-0.1-NoG-32B.i1-Q4_K_M.gguf"], "file_integrity": {"ARWKV-7B-Preview-0.1-NoG-32B.i1-Q4_K_M.gguf": {"size_bytes": 5117561504, "blob_id": "841c66e3242e2a7e1b64641e1ff92248b2650129b196dc3e27c016f541cb0059"}}, "selector": "Q4_K_M"}, @@ -494,6 +556,7 @@ }, { "family": "bailingmoe3", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "SC117/Ling-3.0-tiny-abliterated-APEX-GGUF", "revision": "b923d16fcf28261f12be9ece2b520ed442403f70", "files": ["Ling-3.0-tiny-abliterated-bf16.gguf"], "file_integrity": {"Ling-3.0-tiny-abliterated-bf16.gguf": {"size_bytes": 15803475200, "blob_id": "8c2ea2ead488a0e00bcc02c41fd144d6226ef70eb626afcd589e3960ce39fc85"}}, "selector": "BF16"}, @@ -503,6 +566,7 @@ }, { "family": "chameleon", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "mradermacher/AA-chameleon-7b-plus-i1-GGUF", "revision": "5e13750981e775965061feefb701f4b6e1e1f7d9", "files": ["AA-chameleon-7b-plus.i1-Q4_K_M.gguf"], "file_integrity": {"AA-chameleon-7b-plus.i1-Q4_K_M.gguf": {"size_bytes": 4274466400, "blob_id": "6af766cf8ac2c729c06c430eabc506e32ce10d237c92b426742b3b4cceabcd37"}}, "selector": "Q4_K_M"}, @@ -512,6 +576,7 @@ }, { "family": "deepseek32", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "sszymczyk/DeepSeek-V3.2-4Layers-GGUF", "revision": "54c8e50a9bfaced560a51eeb63cace952e3ea247", "files": ["DeepSeek-V3.2-4Layers-Q8_0.gguf"], "file_integrity": {"DeepSeek-V3.2-4Layers-Q8_0.gguf": {"size_bytes": 16131110560, "blob_id": "a7587dc5291aab8ce6d806c1046a169c499fb39008d4188acbca223fd6ad686d"}}, "selector": "Q8_0"}, @@ -521,6 +586,7 @@ }, { "family": "deepseek4", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "huihui-ai/Huihui-DeepSeek-V4-Flash-0731-abliterated-GGUF", "revision": "a8dfba9c1e43bdf324ee2c7787ed01c70975ffb4", "files": ["dspark-abliterated/dspark-DeepSeek-V4-Flash-0731-Q8_0.gguf"], "file_integrity": {"dspark-abliterated/dspark-DeepSeek-V4-Flash-0731-Q8_0.gguf": {"size_bytes": 10896057440, "blob_id": "6575853d1c3736c160101bc7cd117c8edd39ca847cfdf2273d9a344108edfaf8"}}, "selector": "Q8_0"}, @@ -530,6 +596,7 @@ }, { "family": "dots1", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "Robertp423/dots.llm1.test-Q6_K-GGUF", "revision": "a0404996541e6505cad00b7e3bfa4c7447f4f501", "files": ["dots.llm1.test-q6_k.gguf"], "file_integrity": {"dots.llm1.test-q6_k.gguf": {"size_bytes": 14302555328, "blob_id": "2789482e96ffe36999a3c32c1224f0747c030f7aefe576a3a8510a0aa84fde7e"}}, "selector": "Q6_K"}, @@ -539,6 +606,7 @@ }, { "family": "ernie4-5", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "Arm/ernie-4-5-0-3b-pt-q4-k-m-llamacpp-raspberrypi5", "revision": "86a80f7bb8780cdc73bed0ffbaab3f6ed98c8b63", "files": ["baidu__ERNIE-4.5-0.3B-PT_llamacpp_optimized.gguf"], "file_integrity": {"baidu__ERNIE-4.5-0.3B-PT_llamacpp_optimized.gguf": {"size_bytes": 240645504, "blob_id": "86202506dbbffaddaaa615c932676d3e4c21d1f93e11cd1da3ffb9c6d10cbaed"}}, "selector": "Q4_K_M"}, @@ -548,6 +616,7 @@ }, { "family": "ernie4-5-moe", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "mradermacher/ERNIE-21B-A3B-Claude-4.5-High-OPUS-Thinking-i1-GGUF", "revision": "51b8fa4872193aee98ddf4ae60c63aef52e2a07a", "files": ["ERNIE-21B-A3B-Claude-4.5-High-OPUS-Thinking.i1-Q4_K_M.gguf"], "file_integrity": {"ERNIE-21B-A3B-Claude-4.5-High-OPUS-Thinking.i1-Q4_K_M.gguf": {"size_bytes": 13245833216, "blob_id": "5658d5e0b82578ea7ec0a40d022cbed31013dc6bb0da11c11d16261915b7bcf3"}}, "selector": "Q4_K_M"}, @@ -557,6 +626,7 @@ }, { "family": "gemma3n", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "lmstudio-community/gemma-3n-E4B-it-text-GGUF", "revision": "48ddb4928f7910916a599054277dac92ab70201a", "files": ["gemma-3n-E4B-it-Q4_K_M.gguf"], "file_integrity": {"gemma-3n-E4B-it-Q4_K_M.gguf": {"size_bytes": 4237063328, "blob_id": "7fcb647151fa19a0750538672cf824ef6cf18f74bb86ebe5592e1ed59b4070a0"}}, "selector": "Q4_K_M"}, @@ -566,6 +636,7 @@ }, { "family": "gemma4", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "HauhauCS/Gemma-4-E4B-Uncensored-HauhauCS-Aggressive", "revision": "45b6a334b4bcd1d7f37179df58b3b1d66a184e5d", "files": ["Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf"], "file_integrity": {"Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf": {"size_bytes": 5335285728, "blob_id": "d0027dd3a9128d9323e9f282c8bf010a8526c46477584535991dc1a869b56e96"}}, "selector": "Q4_K_M"}, @@ -576,6 +647,7 @@ }, { "family": "gemma4-assistant", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "RachidAR/gemma-4-26B-A4B-it-qat-assistant-q4_0-gguf", "revision": "64fc324e6829824a37518f5a7c8841ac9a559d5b", "files": ["gemma-4-26b-A4B-it-assistant-Q4_0-q4emb.gguf"], "file_integrity": {"gemma-4-26b-A4B-it-assistant-Q4_0-q4emb.gguf": {"size_bytes": 251938176, "blob_id": "6195511a1bf402b1ab3ec3631347f310b4085774a17a0c6e673ea8988cc0a9fc"}}, "selector": "Q4_0"}, @@ -585,6 +657,7 @@ }, { "family": "gptneox", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "mradermacher/gpt-neox-20b-i1-GGUF", "revision": "aea1d77733d7cef1ccc4556c63666d17e5f95a28", "files": ["gpt-neox-20b.i1-Q4_K_M.gguf"], "file_integrity": {"gpt-neox-20b.i1-Q4_K_M.gguf": {"size_bytes": 13135032320, "blob_id": "1a35c13535c26bb925a2379d9f021de67a0390124ffe2051aa94412002652d6d"}}, "selector": "Q4_K_M"}, @@ -594,6 +667,7 @@ }, { "family": "granite-swa", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "emredeveloper/granite-swash-3b-a600m-GGUF", "revision": "d20c88f11ef8622464f9678e6437ed9e72a07dcd", "files": ["granite-swash-3b-a600m-BF16.gguf"], "file_integrity": {"granite-swash-3b-a600m-BF16.gguf": {"size_bytes": 6047918368, "blob_id": "1961c3d8eb4e97536a2b95ebe0bd6830d508031443e81124516c05505af8c615"}}, "selector": "BF16"}, @@ -603,6 +677,7 @@ }, { "family": "granite-moe", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "sinatras/granite-moe-1b-split", "revision": "3beccd31cf6d51f94c93007e56a21e0df41a101a", "files": ["Q4_K_M/granite-3.1-1b-a400m-instruct-Q4_K_M.gguf"], "file_integrity": {"Q4_K_M/granite-3.1-1b-a400m-instruct-Q4_K_M.gguf": {"size_bytes": 821847360, "blob_id": "3a2ec1c2a78cb29d901e29bbf5162dcd03381e13803d2cbdcff838d4d08142eb"}}, "selector": "Q4_K_M"}, @@ -612,6 +687,7 @@ }, { "family": "granite-switch", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "barha/granite-switch-4.1-3b-preview-GGUF", "revision": "1537da1c12cd890ef0e632678221ecd663485d4e", "files": ["granite-switch-4.1-3b-preview-f16.gguf"], "file_integrity": {"granite-switch-4.1-3b-preview-f16.gguf": {"size_bytes": 8428497696, "blob_id": "3c5edd7df00fa175defc73b6f4af58c9b3d66bfc091ade885d3acdd4ded87510"}}, "selector": "F16"}, @@ -621,6 +697,7 @@ }, { "family": "grovemoe", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "mradermacher/GroveMoE-Inst-i1-GGUF", "revision": "4e342287d10e5b616e087509fe2a979293bc7b0d", "files": ["GroveMoE-Inst.i1-Q4_K_M.gguf"], "file_integrity": {"GroveMoE-Inst.i1-Q4_K_M.gguf": {"size_bytes": 20167310304, "blob_id": "e1d8ce9e762f74e65d85bea1e8f8e06db70f2dd735cecee9a6d889c71e6fc010"}}, "selector": "Q4_K_M"}, @@ -630,6 +707,7 @@ }, { "family": "hunyuan-dense", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "bartowski/tencent_Hunyuan-1.8B-Instruct-GGUF", "revision": "bc16eeb7f561c3d7937aa2cde457668695c237ca", "files": ["tencent_Hunyuan-1.8B-Instruct-Q4_K_M.gguf"], "file_integrity": {"tencent_Hunyuan-1.8B-Instruct-Q4_K_M.gguf": {"size_bytes": 1133084864, "blob_id": "558ae86fc4251fd73cd54685319a83e7550997537e3ad41043ad5c0b5a2a26d4"}}, "selector": "Q4_K_M"}, @@ -639,6 +717,7 @@ }, { "family": "hy-v3", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "mradermacher/Tencent-Hy-30B-A3B-uncensored-heretic-i1-GGUF", "revision": "6a7a7d5bea1641ec15202f14d4b467d45c104f06", "files": ["Tencent-Hy-30B-A3B-uncensored-heretic.i1-Q4_K_M.gguf"], "file_integrity": {"Tencent-Hy-30B-A3B-uncensored-heretic.i1-Q4_K_M.gguf": {"size_bytes": 18236703520, "blob_id": "76d19eb68bea4de46f2f3dfc82cfa6896da447b9aec91d7c6c3e7791f2d306ac"}}, "selector": "Q4_K_M"}, @@ -648,6 +727,7 @@ }, { "family": "jais", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "mradermacher/jais-family-13b-i1-GGUF", "revision": "0339532e089ec7b6dd502eee16c758fc324046c3", "files": ["jais-family-13b.i1-Q4_K_M.gguf"], "file_integrity": {"jais-family-13b.i1-Q4_K_M.gguf": {"size_bytes": 8967115424, "blob_id": "8783a7d62b714c8520f2b03155f7f52145714dd9006653b7a64a6a6a097f5407"}}, "selector": "Q4_K_M"}, @@ -657,6 +737,7 @@ }, { "family": "jais2", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "inception42/Jais-2-8B-Chat-GGUF", "revision": "61365d41aad6a2ffa9d7b7f79dfce21b65833254", "files": ["Q4_K_M.gguf"], "file_integrity": {"Q4_K_M.gguf": {"size_bytes": 5104022560, "blob_id": "26aef0cee0f9960cb12cc74ffa1049ca5e35e8159621aa1bacd9f9f703239923"}}, "selector": "Q4_K_M"}, @@ -666,6 +747,7 @@ }, { "family": "kimi-k3", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "murillo2000/Kimi-K3-0.40B-GGUF", "revision": "88de02cf8fa37f87eb06daaed370ac9c3411d5ca", "files": ["Kimi-K3-0.40B-F16.gguf"], "file_integrity": {"Kimi-K3-0.40B-F16.gguf": {"size_bytes": 784318432, "blob_id": "411c197b503e6fb9199a2b22115e32dc4e2cad803fb112b24967737b3bab26c7"}}, "selector": "F16"}, @@ -675,6 +757,7 @@ }, { "family": "lfm2-moe", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "noctrex/LFM2.5-8B-A1B-MXFP4_MOE-GGUF", "revision": "6f9e38da4a1715057e3c3a45105037944e767117", "files": ["LFM2.5-8B-A1B-MXFP4_MOE.gguf"], "file_integrity": {"LFM2.5-8B-A1B-MXFP4_MOE.gguf": {"size_bytes": 5138197824, "blob_id": "a492e8b1d8c201b90de421f0ea1b6ed478aa7f4948604ea6659cd9afca715f3a"}}, "selector": "MXFP4_MOE"}, @@ -684,6 +767,7 @@ }, { "family": "maincoder", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "mradermacher/Maincoder-1B-GGUF", "revision": "1c963c98dfb478ea3b4719299bd85fbb5cf30899", "files": ["Maincoder-1B.Q4_K_M.gguf"], "file_integrity": {"Maincoder-1B.Q4_K_M.gguf": {"size_bytes": 672108288, "blob_id": "2db4b53955a58f4483442110fdd5534bfa22a45447f35a131b839a034ffa2998"}}, "selector": "Q4_K_M"}, @@ -693,6 +777,7 @@ }, { "family": "mellum", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "yuxinlu1/Mellum2-12B-A2.5B-Claude-4.6-4.8-Opus-Thinking-GGUF", "revision": "a9c7b1a32ead28d4a868ff31bbf9c15a97362cfe", "files": ["mellum2-claude-Q4_K_M.gguf"], "file_integrity": {"mellum2-claude-Q4_K_M.gguf": {"size_bytes": 8071294784, "blob_id": "2086900d9374d7a12388b6a44f7d34afcf5817abcba7983dc733d38c5003e2ea"}}, "selector": "Q4_K_M"}, @@ -702,6 +787,7 @@ }, { "family": "minicpm", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "openbmb/MiniCPM4.1-8B-GGUF", "revision": "ebb834d0ad9acfa98bd16bc963ec51be5f7e08c1", "files": ["MiniCPM4.1-8B-Q4_K_M.gguf"], "file_integrity": {"MiniCPM4.1-8B-Q4_K_M.gguf": {"size_bytes": 4965526048, "blob_id": "9d2ffb9145bf7a88ddb94b2542b42d925293666c42962aa158eeb62fc9708654"}}, "selector": "Q4_K_M"}, @@ -711,6 +797,7 @@ }, { "family": "muse-glimmer", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "Blackfrost-AI/Muse-Glimmer-30B-Abliterated-GGUF", "revision": "83e7dc723f5515704adc8767dfeb279de3df914d", "files": ["dflash-Muse-Glimmer-30B-Abliterated-Q4_K_M.gguf"], "file_integrity": {"dflash-Muse-Glimmer-30B-Abliterated-Q4_K_M.gguf": {"size_bytes": 1631210720, "blob_id": "1b58b92b670d20b49d06a9a1210e4bf5350696bb8f7b4931769b7000d0456eda"}}, "selector": "Q4_K_M"}, @@ -718,6 +805,75 @@ "execution": {"trunk_layers": 5, "mtp_layers": 0, "activation_width": 6656, "boundary_sweep_period": 0, "speculative_policy": "mtp-if-present"}, "resources": {"runner_role": "family-certify", "cache_policy": "immutable-local", "estimated_model_bytes": 1631210720}, "notes": "Muse Glimmer dflash text and projector lanes" + }, + { + "family": "nomic-bert-embedding", + "class": "embedding", + "profile": "workload-oracle", + "cadences": ["manual-full"], + "artifact": {"repo": "nomic-ai/nomic-embed-text-v1.5-GGUF", "revision": "0188c9bf409793f810680a5a431e7b899c46104c", "files": ["nomic-embed-text-v1.5.Q8_0.gguf"], "file_integrity": {"nomic-embed-text-v1.5.Q8_0.gguf": {"size_bytes": 146146432, "blob_id": "3e24342164b3d94991ba9692fdc0dd08e3fd7362e0aacc396a9a5c54a544c3b7"}}, "selector": "Q8_0"}, + "evidence": {"fixture": "scripts/workload_fixtures.py#EMBEDDING_INPUTS", "comparison": "batch and individual vectors within 1e-4 coordinate error and 0.99999 cosine"}, + "execution": {"trunk_layers": 12, "mtp_layers": 0, "activation_width": 768, "boundary_sweep_period": 0, "speculative_policy": "disabled"}, + "resources": {"runner_role": "family-certify", "cache_policy": "immutable-local", "estimated_model_bytes": 145389792}, + "notes": "Local full-model embedding, HTTP and official SDK smoke, and pinned CPU monolithic batch/single-vector parity; staged tensor filtering is intentionally unsupported" + }, + { + "family": "jina-bert-v2-rerank", + "class": "rerank", + "profile": "workload-oracle", + "cadences": ["manual-full"], + "artifact": {"repo": "ggml-org/jina-reranker-v1-turbo-en-GGUF", "revision": "607d8664c787e517e5d6e339d21f680f9002c931", "files": ["Jina-Bert-Implementation-38M-F16.gguf"], "file_integrity": {"Jina-Bert-Implementation-38M-F16.gguf": {"size_bytes": 76971168, "blob_id": "71abc010bb3dce97812ee971509a5cb6ff6f6b8cfffd8480129242f605521fca"}}, "selector": "F16"}, + "evidence": {"fixture": "scripts/workload_fixtures.py#RERANK_DOCUMENTS", "comparison": "scores within 1e-4 and identical document order"}, + "execution": {"trunk_layers": 6, "mtp_layers": 0, "activation_width": 384, "boundary_sweep_period": 0, "speculative_policy": "disabled"}, + "resources": {"runner_role": "family-certify", "cache_policy": "immutable-local", "estimated_model_bytes": 75293188}, + "notes": "Local full-model cross-encoder rerank and HTTP smoke with pinned CPU monolithic score/order parity; staged tensor filtering is intentionally unsupported" + }, + { + "family": "t5-encoder-decoder", + "class": "encoder_decoder", + "profile": "workload-oracle", + "cadences": ["manual-full"], + "artifact": {"repo": "Felladrin/gguf-flan-t5-small", "revision": "d71c51f67519edd3154527c2d8f20288bdde9705", "files": ["flan-t5-small.Q8_0.gguf"], "file_integrity": {"flan-t5-small.Q8_0.gguf": {"size_bytes": 113709824, "blob_id": "f7f769c360b1ba830b10dd3b7e7d146dbcc4d487962be7dd806d7d52e0a9c2f0"}}, "selector": "Q8_0"}, + "evidence": {"fixture": "scripts/workload_fixtures.py#ENCODER_DECODER_PROMPT", "comparison": "identical normalized greedy text from pinned llama-completion"}, + "execution": {"trunk_layers": 8, "mtp_layers": 0, "activation_width": 512, "boundary_sweep_period": 0, "speculative_policy": "disabled"}, + "resources": {"runner_role": "family-certify", "cache_policy": "immutable-local", "estimated_model_bytes": 112678400}, + "notes": "Local full-model encoder-decoder generation and HTTP smoke with pinned CPU monolithic completion text parity; cross-attention staging is intentionally unsupported" + }, + { + "family": "paddleocr", + "class": "ocr", + "profile": "workload-oracle", + "cadences": ["manual-full"], + "artifact": {"repo": "PaddlePaddle/PaddleOCR-VL-1.6-GGUF", "revision": "511b09642bb324401f15f97cc23bc67e8f0a291d", "files": ["PaddleOCR-VL-1.6-GGUF.gguf"], "file_integrity": {"PaddleOCR-VL-1.6-GGUF.gguf": {"size_bytes": 935769056, "blob_id": "f3ae46ec885050acf4b3d31944431e1fd90d50664fb09126af4a3c050ba14ee8"}}, "selector": "BF16"}, + "mmproj_artifact": {"repo": "PaddlePaddle/PaddleOCR-VL-1.6-GGUF", "revision": "511b09642bb324401f15f97cc23bc67e8f0a291d", "files": ["PaddleOCR-VL-1.6-GGUF-mmproj.gguf"], "file_integrity": {"PaddleOCR-VL-1.6-GGUF-mmproj.gguf": {"size_bytes": 881770560, "blob_id": "204d757d7610d9b3faab10d506d69e5b244e32bf765e2bab2d0167e65e0a058a"}}, "selector": "BF16"}, + "evidence": {"fixture": "scripts/generate-ocr-oracle-fixture.py#MESH 42", "comparison": "normalized monolithic text parity plus independent MESH 42 label"}, + "execution": {"trunk_layers": 18, "mtp_layers": 0, "activation_width": 1024, "boundary_sweep_period": 0, "speculative_policy": "disabled"}, + "resources": {"runner_role": "family-certify", "cache_policy": "immutable-local", "estimated_model_bytes": 933384192, "startup_timeout_secs": 900}, + "notes": "Local full-model OCR and HTTP smoke with the pinned PaddleOCR projector and mandatory generated-image monolithic parity; projector and trunk remain colocated" + }, + { + "family": "qwen3tts", + "class": "speech_synthesis", + "profile": "workload-oracle", + "cadences": ["manual-full"], + "artifact": {"repo": "ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF", "revision": "ca27d74bc954b73dadab5b71ca265d87fc861a7c", "files": ["Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf"], "file_integrity": {"Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf": {"size_bytes": 1847874400, "blob_id": "ac7931aeb2e7aad1a6ed6602d353a5679c9d096b18ce8204ac730a8408d572e1"}}, "selector": "Q8_0"}, + "mmproj_artifact": {"repo": "ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF", "revision": "ca27d74bc954b73dadab5b71ca265d87fc861a7c", "files": ["mmproj-Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf"], "file_integrity": {"mmproj-Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf": {"size_bytes": 446422912, "blob_id": "6fd65188839bcd6ecc91b277ad471e22a0edfada4699a0fe82f1165c18cfcce2"}}, "selector": "Q8_0"}, + "evidence": {"fixture": "scripts/skippy-tts-oracle.py#PROMPT", "comparison": "fixed-seed PCM format and length; RMS error at most 2 percent and waveform cosine at least 0.9995"}, + "execution": {"trunk_layers": 28, "mtp_layers": 0, "activation_width": 2048, "boundary_sweep_period": 0, "speculative_policy": "disabled"}, + "resources": {"runner_role": "family-certify", "cache_policy": "immutable-local", "estimated_model_bytes": 1841844224, "startup_timeout_secs": 600}, + "notes": "Local full-model speech-synthesis and HTTP WAV smoke with deterministic pinned monolithic PCM parity; distributed staging is intentionally unsupported" + }, + { + "family": "ultravox", + "class": "speech_recognition", + "profile": "workload-oracle", + "cadences": ["manual-full"], + "artifact": {"repo": "ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF", "revision": "5390c7c41cbd6f261f7f205fc0c5ae61bbdca650", "files": ["Llama-3.2-1B-Instruct-Q8_0.gguf"], "file_integrity": {"Llama-3.2-1B-Instruct-Q8_0.gguf": {"size_bytes": 1321083008, "blob_id": "432f310a77f4650a88d0fd59ecdd7cebed8d684bafea53cbff0473542964f0c3"}}, "selector": "Q8_0"}, + "mmproj_artifact": {"repo": "ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF", "revision": "5390c7c41cbd6f261f7f205fc0c5ae61bbdca650", "files": ["mmproj-ultravox-v0_5-llama-3_2-1b-f16.gguf"], "file_integrity": {"mmproj-ultravox-v0_5-llama-3_2-1b-f16.gguf": {"size_bytes": 1371123616, "blob_id": "b34dde1835752949d6b960528269af93c92fec91c61ea0534fcc73f96c1ed8b2"}}, "selector": "F16"}, + "evidence": {"fixture": "ci/llama-canary/fixtures/audio-smoke.wav", "comparison": "identical normalized transcript against pinned llama-server; fixture is unlabeled"}, + "execution": {"trunk_layers": 16, "mtp_layers": 0, "activation_width": 2048, "boundary_sweep_period": 0, "speculative_policy": "disabled"}, + "resources": {"runner_role": "family-certify", "cache_policy": "immutable-local", "estimated_model_bytes": 1313251456, "startup_timeout_secs": 600}, + "notes": "Local full-model audio-to-text and HTTP smoke with the pinned Ultravox projector and monolithic transcript parity; unlabeled WAV does not establish transcription accuracy" } ] } diff --git a/ci/llama-canary/fixtures/audio-smoke.wav b/ci/llama-canary/fixtures/audio-smoke.wav new file mode 100644 index 0000000000000000000000000000000000000000..41effdd9be8fa7990d7b4060176fe36cee008041 GIT binary patch literal 29072 zcmWh!1#sI+6P9F|9mk2|Ft=gmq)7u!n{t~nGcz+YGc&w0Gvg~Wx36u;EK9NsGF$Nb zAB|_uGamW0y1U)|pnLn8nzhpapiforHlruajFkZZ0Q#<_<-Wlp0Du4)(5Zcop5eZ) z?Yq=;ZQrA&r=M@!Y1r)1g*ll;nK{|n05Edc%wes3cd@XIEC67vo~;GO09}EX>@ao> zo6Fu{f3k1bdu%zoh`GybWLpAhz;^I4cpRJolmW8<54#Bb0=57<0nuy>+XhfF>Fijb zBk&G*$&6(+>}cQJK=vEc3Ahf_fQ4)u#z0?T1^_1JH9HEBgV%tq%us-0M9@j_9CL`B z!_vSeR?cn%dINnxJLmxJvT5KrAOcKck1>r*9sQmi#f)U<1A#y#Yh&iKYIY_#2{;XA zfK5O@Xge?t415L_0yo$jOffT&?m{18mjYwh)l4iP=R^ZyC|{QH>%neNEij244Eg~a&T^Il?tnJB zk(tPxqpMh)J`0Qk%9t#`0DS=d0QcEP?0t4It72aOHu@6)f~CL$;3jYm5U}T&*5DeJ z4?JdnvmwkL`XQUozGhVHWVVrR$zW_dHkn<{+y&;bKfr%LDPRXqve^L5E@WDAkHZSC zKYSg`hWbMTq0JBp{9s`qiTOZXrzbF78GsQoV()cNF@2TU2>i!DP$_36rQavK>(8muPk8{0GT2RYhy&~}o{wXPyv zWC|H!L+lsb4W5ql5EcP0aK!urqGOUD(xQOepx%l>ilpG}!QVslq4{C)VHZOmh8$KL zmrDaj$bR@&NQX+t`Mngs65bbF<*(%J<~DKPa6NDVe1-EI%wT8x-b)@Mr%t;AT&tW9 z?UXgcvfV5&DToNSSnhI!gknsmKe|3nMxl)CM@Yr0g!c_d4BNjKH-23e`I z>7MITbrl+}I=M;FbWUGw^*9C87wD^?RP;^K7&s&Bd`#Em$SiI)KjTGeZAwnc#MHv{ zh>SDoY_cu>PsAcchk(q0oWT15<78|6qa+*nB)5T=z`M$uD{@Qbh|cjJ^NP5?q3zID zhz3fTH`F_i+5X7PHSHo=;B5_R{e0bVO{LnTEN+Ty(liZH(oNHpVX7SUb+uG0(becq z=)6c5!#4eTz1aX5w&*VCzUy~uL)34S;~G~t#ORmVr!(t8xnOg^)8IK_?PFFZuFfbe zVk*{Go-A3LzdYYw@H2mM?uYEFS(0?LS(nhge$|33VWVh*z|MIN)j=}>Bb7~cqm@7^ zPb@snO@Qt}1K=NE5KAyQz$7NiJJ`9Od_*MTzp(D;R_#pn49#}+uEwN>)W)-oNW=5S z3CjM;GUXuk3@xI0sa=7rGZY)ruyC9~YcUF6iG4JzL$lF=`h~hly2HxHbvf!EWNQF{ zPlz`JPm4;AD@!tDT+KUE2DBU6@nx&S#pT8Sw)nRoKeI05MCR1w}z^x9B15M$}xwS9vf-n4kFaJ z9PNybMvIZHn!8P4qe%DMJ`RqMlmzaHC`>q%Iys{@*Vv+GYp7#Qr!O^!S~5jL3uANt zO|B4SK$iPI*3 z^9hsF^uU~I-eAf$X$iOS1-=|pqKgdA_3yM}RVy1x)ceV?+}_fGLB{BwsiU(-=hPK+ zE^l6Qr-QIFxAVu=XwmuXYgx6aPh|^#1coq0T z*VE^Keb7YcH3))PYy|bk)!e?zQf3-R9K{8w0J*B)qU)jQsa~ovX!M#y9jUKD4k5b? z%}@?jjsM2K8C9k_^Dnc;ve(+xidnmoJ*n}v z>8FNG8-PK6&lJC+Kd0p6EX&svPAjWu#k4WDo7%Cc?ff!FE|ak&d1c)1h{Iv(kn{4- zl25!Wjtvk2CxKnyap)(^a+krgp>XIWr~zKHd)bj}SK8m5;pl5SV?KqoG^mkt`Ue`b zYPvc?Q>$vGY^ILX?$GSiywROT?jbvoj|LjMj+59uyxL?lEgiXu7&Uy#2G^bc$&%kj^b4Upk}*Tr?B8sPz6HT)P_ z15RO+sKM?I&OSEEe2r+pccJHz@%oOs51PShLe)*JR6SA_sm`g7s6VI?S z+%oVR@1~@mye#Zj z;H{1Aj4U2yTwokwJc#eXIG7n#;KiozWJ}j&?>A-yRLT1y9O<__;H~0M*p;Yeah&Ec ziKe80)T8N3v#N6YNk{+`55v<#l;=d+UsKt?Hfi zbc4O|Ueh?`2-PHYjOM=Pf%dcRm~S^UB1a7c=umV8`VDP^weijQ8>&NRqj`opJ+8+L zFY$2mEz;{4=pO0a$Z$EQ_>;sN{4WO`3u_xy9ecEyBaxqaDr0Ky_##(}y0WR|4=R!? zUsWvdU9ZZqvZXEV76JKFvKFOM2}B$b6&juuEDwC<#}gL8PuO+di_U0UgIQ%{(F~+a zt5oTldNd|C%&c!+pH#1?|5oR&>sY_HKCaMgq6+K=i~^N9tscKUpDBD`R zF28>kH+4|L>bN~oXRUcH7)!j5|O?T~CZKm$9PO874PeG<3#}F+NYIu!|L6#t! zkwWCYewV&NzfSi+TdQ5IS*S`GB;b)#h(az6*eyF zb8JcTiAfhzQ?kzGFDQ9c_P2a=rJ~BO%2tV1Zm676ajs=V>6M~BdFwOJrPMd?6+0ku zZ|M4Z|I6x?FQs!)i8Z^L0;j)%rL37RW;63339N<%_=@LG;`8`TFO&3f*~a zjCQm}r0%c0*3?<0(%K9&j3dq8tlw=@o&S0#gTHv)#6sB&#encrQGH`?Hye>;P7TWH zn_pUzSGJ?&hjOeUvr=2}p+a3jl;13SUox#QKesTWEctG;busfImV^wH|B^~Y<=jO; zG*#sKXj^R=K^(%INON5`br+?&@k+zn`g`>uzNaW`*wGNy*uQZ}V|mj6-}~6$TTfM* zcN$#NTbt~w#4z1xJ*)4FEJ6+dW;W-9X(fEmwO@%~PFFUQmzG zhhsHFPs=58roGlxLJK%@!4`=oV0{P~!9>gBe`ao--U)Ttvx3kv&p^Ye~m`88$xI=sDCN%Hib6rXk6NO zqtV*v*VL-%i|iwy?xgO8E=PY~pMVTT)*=W38Faq9w-gzUz{pGe ze7!>7OLt70r0u6stKyZCrhQE*>N?#O^tiFk^vu%97VN5~e?V&ff0E+B=^;BK&PTtA zTaYk1o=s}-*+aw;n;!z+-olopQ)JLDE*Oi9jd9v!QXxD~QaZk2u$o`=Q2JMRZ)itVlW zl<_EfSbspHSN>{j-w<9uv94F$*198g=jyWSJ@q>qem6X6j8eLk<5Z#QOwA7srs?Y2 z->-CPT`Ro;nThZbTwjWGLT>0KdVyY}3)el-tWi%=?NUao?r4T0$FO?iPgAXRiT$j* zipk~7798>GAGk4iPIz>5*SMF>C#1-;dgkXAw<(P%H&@)L99%W8RdV&*R>n$c#fq|$ zVo_dB=7|)%xg@qJ;&%vNKFP18U=PR14Ds}F6p+4LgfBO|(iLeADR(sXZTMT?%a`p2 z)(16=Ynag>YYb}2Q{GcnsV1s>Ywl?5njOAfF;Le@cV1VZpQO*$*XaWE$@<5-QeA`g zp|-1biRPO+QoTwwLN!fwQZ-C{So_rAB&_5uR}XqNxRE?F7*N)u#ku0kg-h}u=3dOM$#|K3pxM3XvtbW|KFL=5Z5IXc z+jEw&0QKIr%l_NC$DD3@Zp^^PpsSIGy1Cjz8j&VNJzG_w3j2QqmFdb8%B9L8rK@SU zQm)#r;;Dnxe|=BG(FW)`=(_4g>F(-x7_J#sA~M50bUrFEv`2a)N05#B_1Ynt11hvh zr`>0*WgB@DB?IN6h|RGB60$SF5~?Dj)ty!gD+Q&U3RdT73VEL z652boF!W~#6e3rAm2LGqA<37_71ayQ^Y8L@!Eb;`RGoXS>w>+XMQMz{)6lEBPU<+- zZ>2!lq3K#vuxgPiUiDYqTi+TH>N_G!uv3^BnS)HiuHc#Ie8gaoU}p{MkgtYb*fNxl zX5o3pDvUw{jj(Yey5Fz^xvJqRL-ntm^@2A6pF%psL?y(e+{#&5*0H*-&F*%6+WoC4 z%$b;UJaK38l7zl77eg-uG!q4LPeM1r_s~54YQLBM`}{VDfBKybY_GT)fMlqb>c3a6`TdYN%|;Y8=x*)@^7MX%^}fI;TF}*v6b?9$~4rwI?%8pNJ{u z_e3_f7&ZIqATu%-X@L^Hj!}odH6q3X#BX!5xzv

Spdpgdua(Ihcjcmfeon6F(wx zU79dYS@Ngl-&UL2R=3+!8J_bp))KZi!V)10QThKCj)v9jKo&#}5&4SyPjHWp|WBliu%3{UYy zi-6o~Jwh%cCzx9hMq{}Vz@iOi1B!;Bi}ev`wDF4(!+#Uy*5>3|%O!IwawZvS9%1Am znC`v(i2tFO4$ax*i5VAjKNpsjZYf`0?XI?$b;-CJ{y=)x-z^^=xJK;HT?kHN@+pt! z68l7WFX%<+hoG$iN9F0EOlZsCe*zCk_VC`)H*B@|8v~B0wYyX!8t(sv{*G-pqg1KC z>Ief%95J^v$69vS6|U3vY+JmwqdCmD0^4PX^<^Ui(g%q`v#}XCgMYS6BOj4X*5(eO z6R<6}&LNLmw(39Y!l>V2)+8`xLq=|1e1WkjWiI0QJ1r`3=Mdh5? zU>mTEhCKPy18$#yei7}Xi^2p!ZG#)at_1fDIP0eotYGqNQJ6#fLdB{{!N+_ zupF#mw{g!3k_8gpG+r|<2jqIsGuuR)Las)=k1C2-9P&r8JIEBUKr))Y3hL%rN&KUp zr5vo@qhm4@90g$~oc zHuNHrut5gY1lUWgFU$|^Q(XD>Q?>!5idcqRv}_k1YQ8row%|^|pn}o)0r`LOq6z|v z>T^tS0sbohl`kiD48;IW8zPQ?+uKT4_6rdM({eq&w;V_ z^>~OTw4rUoziM0^sk)$y&|KAK=tt>m@RjE24@X|08_T1f{v!g1Si-s;5uK@G0v^02fX1y73fMRyAvc~5e&oTJ(2 zvqh9h3bU93M|ragg~tK$dR@>d$oP8P2k+>++lrddTSm>q#00s zJG2f9!I$B2rlpn*WFKFoPuVk^J)B2L55B{Ak3AIjB1_Uj-(q`l`y#wxbxwWOtgJoh zljEBQ+-APpW;?Ac4u61C;T^m+a0R=9zQpO^&x8+(854F={wQE>z);CbUNuz8Mml>K zC#yd+q&G&W{wPb;3$=ID-PKzC3$)DepYf`FpCgWZWxMO_@0@I(>1bx}Yo&Zj_S`NQu41dB!=^p>y1Fw{(@-80!w}OQISZ zhp#X`^PxEZm{waYJ~YH-Tj)|brS_TRUrR6Qq_=cPxC(GGx0qPyxNv^Z=m<8fZP0eve*b%Z5}}*-k#mZgWlhCE zZK)DzoY-_3)LWE!4oiY77pUq@%hc!%88!*$=8>s(@MGI@zsrd~!q z{s-S|vYE5Z=goh~Rkpu2yVK@aVAfiOb3R2(N}rH-ys&*qUh&m}t@&T`;&Q0;_RZ9R zb2($ZEnI8ePpN@yD1fsM=y19(Gk`PBZ%Xi<(6%9+g5qSmq)+^Qijo9%oVL_8@-MbQ zTd4-r$CT@pEtH!Z$2IJ3TCBdO`;7K6<&X<(92;R9;Oyw?<8(Qy9SOFp=0QX*!8H!T zN#jJ*5u$^sjag(#B`?`l`s#R*xsQ9OUqRfTjH3MM1;+fH1ziej3lsAVS#wi5MW;)5 z0n&TXy&Zs4V83JU?i%2HVv(AbnF!*p4_-+%Qp7&f zN}`=o|@yAo&XB6io$m?jMo-w+Ii<0*&DXdXzMKxZ4i8m?&* zmC~kl%BiYDYOOZWa1_6RAHcg4&CLBQ6Rbt#D$5{K2V$QwAMc1Q#oOW{tRtRA^flJu z|Co}@gNy|J$XdoslFSMPVlKvYY95->kZI1npWiiqefEjug4ly$e}iQ5k@BPjOYWS)@tG|JsM~C{5pGwq8UNe+``x z0G(mcQ^C9$;{E>3<$r=tM1G9tCD)|AP17dFHQy9l7hN86B-$7`J^Xh_O7M62{J>W7 zNx?wK(V%Gg?SK)oLH-YA=Ved*`iql9Re}PJ4k+<4Bqx|uH|qG$ewBP_iX@g3U+~w4 zcls=3puRzSM7v0rq93a>Yu%ayni{RYZisHGZnAEfZ?sH5Mn6M8#?T47k4?nK7(M1@ z?#bX+?qLBbDUxjuEC@as)+%ycbZu;E%+bh_$k&mtBKJqkju;ksFp?J;8NNI8NASqt z0ikHttMjN$YMW~%OjC%h###6? z^oPNSbkmDKIjnszRsGo>Km3I-+i^`A-d?#g;#|A-3(deeU(N z5W39E6ho4>qHPk;|GR8_V3Yi~{811Zd|7c;elz%X#DVbkA(O+#g=d8>RoE2V@cQtz ziZAk&!5xBT_-~Q+R*aKR^Q-lblrHAwfewD0pcgcT?##>t!rhcjV%z6<=Xh?~WI1JZ z;OjBe@C3v3pY>a@UxpV*xyGtZ(`EXi)>6AdvqWEr73rR;a?usGA$E@~+hz0o^y;X# zz*X=KS0b6^mm+%2Efltu=>w(;Yo7 ztaVLAV$efG6JBAtg6}tA$Vsda;pnds98(KJwswG-kDb;T%u;YK{n9YN7D0c6GN>$9 z6u&EP5-VhD#Jzbv=nn9xkX0euJQeWn0RO-eSzqaMS)`aNiIaQ@ju8zO90?6nbQawe zZ*(2dW7EnXM- zAHO4KnP{4I0B3;sCI@zv@|Fhog+q9~t(d%PKpTo-&jt>QslzrgCnZH;iI4+rNURD7 zz^=M0M3&4z<9Ewf|6zW^MV&MY{7-Ug!E(@h{~^-m zhOXpH!Ki?x%vI~`ps#EzAlw-REQ$EVUUpv=%Ora_3%%#*>A_q<9=D4y4l_pG=l*0@ z(gwIV^pDjk8pyrH+iY>LyW#?zJ;{&Mir`g(cNl1K1o~JoQ=In-^Oir06+PtdGY)n41XSVyj^1Qv zZY6vOzK-<)hVjmcJ5yW};Lp%oI4PExp!v)*XRSnUUmo~Kw~YHvGS8|B^B7Oq;w96p z&AA-|``e~^j?&3OmfB$#Nm96n$vpd!z)}1#j|iQ`PY(Ldn{JhmtHgT+gu}(m;XU+j z@@WZHaL#)*wAY@ZI7&8yhI0nHd%#QR@4_y|EOuvbF*AkWPzyO1q=>zg8O_b2o5OQuQ|4VL9^sC4wF5uM zAAp;*^K4_oU-`YnDhX8x9PmN?AH9XwgrYW6w!o5Uz$`(i; z3eL3+r(klYaALqRVjL$IaDYZ8-Sfe((v!?IdjEnq{dYN5+XLa$z$czsqntyCUyEO3 zGfmfHw4N%@Uz?3N5a?!C>VHZe3T*Ztt|Tx^G286R73M;D9yr}GRq&bwia~KdA z9Li%gnI2nEsW=vC^3Dk$s2O0gqn`%|TuA9{{$iQCHD|5!3%5;RgLWqOCub4#nhil0 zg@Lrn^NCOB=|ORrr;Jo{uLz&P zXI(SUFj0q~2^3B2Q<}39??dHrA?ou4n zt|k{m{jd}CK}VG6F+1D#L>kVn=Jqyib0$U1vOX8=b>8tz7te7#fWNR>pGqW}F>)Ji z2)KpY1Agh5?EP0Tm6S`Y)EH-(bf3Aae1qF-so`3ABe)|R-#yd0gdo>B&iTpT&0cb` zRIb-0U+bu_MT+Ak3eyoflD}U%lOXLI1O8(Hx}!B+KGh!LxTNS|-vFJo>YxkqIb;y9 z#p~f*bz4N^scLVyNMXGJ&l8kUOT7~~Z=GE6{h*qram;(pYu**V=~kAW=NaLH%*+Un z)0;b!UF>%o849rh3$6KHo!~Z*#I197Bc4R8rqaCA@l1)rw=-H}r@X>sfAcMLx_>#x z-!Hjwl60v*VTGW0LqcM%y|?qRYdthFF6hUrfFXj@=2e0`qcXS->T3=cm6`3b@sQZ& zbgUHJ;Kp$0yQiRU14^00{0z=`bf0u7Cz+fDIC<5aBG=blBw_=%Qs=eRz5e?w2;o$yX#u3#U(rDcJnzhi;$ALgaG zz5gg+A9ces1yTsxc-uM}xXFHZ+-7q!*CI0#=YYZVczGIBTDL7=CU=M5LR@D16+W2$ zmzu4&1*A(alKq(s`i-!){VMg?Z;e&Q@5K~(>jQTi5w8bYz^M(ogH#GlwAbwRgMmR@ zvF!sBBi;knGk)fwf^zzvv_Ck*)Wi*-dT{%?2f$wNGkpn~>YmMSU^w<_;bh1GPnqx2lo)x3AdVxsEGmbOq1(_ zrL`=K|C~I*K4YQ4Etts3N*%C@cG(gociGd{u8c@Bj-TW(xvp_+f(ClJYc&+&tnw=X z|M3=aQtbT&%YXtm;J4WOllFI}iIVty5@VC;C8EB(I?Ga^2g~qRdqvc}Wa(j|UhdZ1j`oY}C-Ftf!K&^4q8d?vX^m&O z_yhcn3bHNdUl+ZzCOJUqZ_#XLpkp;OUh40@YrV&B5N)^3@#gb$MH?NPi3#G9etYQN zrh#CRbOJq|#(ZL@F5V4}A+jk@0cA5!V0r|4so$PU&Q#G>F6!byCuj*1=PBT|5k-5~ zk@taJ;sI2pbB|9mR?g+S+Vg6eTMnI%#e(>k9m`!X$HV&v40d$$>GfRPDXi4<2u!5n zVHuDhaDrWJCDLZ@a}eQO;!Oz9yAuT$$$F2!AQAv3|1h0DD!;>VX z-bdacWCnk+s1j~6iEiM_d%;5$A={}fiQelw3DOb zs6ZDun!lD~kU(%>_a^8U_phV_e8f2dO6TQ^-cg$zYWOnj=3k%|Fk<%xN+Ve6(>C}! zuDc#G7dbHV)0OY)&m0H*eO>W8_cM1lkHI;So@@`aLi84o-Z8>iL_+i$z>dHcEp zoVUEy08E#AhO(VS1XBZEgYO7ei$LBnzuAJWf?^pc*UAzFJA`}XoFIp!1Gm7>B@_F4 zy;=Mmegr&^S1D=`@R-hCEvJFw&n|Mervw1)Ip-*KW_#v4#P&$H!&PJcfg8vS$1RdE zH@8eQpCO9P4$B$y7SnF?6RXy;z&P6U#QL1vVr_4EVM}%#a831G^?sxefVa7T;JomW zq$Hq0UZvO+njUc^`g829m?^Ql<7PMO+5BUCN&Na4ZB)0&3tz9&+LBp_$?3PTaPUw^`< zk+@&)tc_(F5RmBI`8t#Z|QqYF3YEliJ3Y>b~2_fI6CfMgEo5OF@|Fnz`|-?`o?J%>t4SWUS8GnA5IsWU;mA zMQMGj9+k@c?&(PxkF(P=dL^uhoT=C^8!ZXvHPHjT-#p2VbaMuV>0WC*hCO&I{X_K` zofzF_*lhS0mEe7e*QT?kVB(ZTCfi1cN28UfPw8o8)oxLeoN1B?05hP!Sq}<{I81!;nAv{)XG=HtZm- z#dqPmh>wDK;~0PVSSU|BBDI7+%@bGA5tOeb!=8`PkyN zsl&oY1kDaR8PX`)&p!4>Bh zrEQIq@u`n99g7`GKYbN>8vc}XFXU$Gdsp4etd;iM;3nOEb+vX^W%q>D%LY1ji|CGe%bj;P329zGm zsEqlpXea+KV3(*IIOCb%{?|Glf1!_5!OA3LG5$mIu>n!XqVw)l+plKRXj-KT9k?FXEG_9TnMeB83zyu^nHh2!Pe8^dU<%y`m4 z{T=z#ddc3|p*6=DH;^6Shw>94JLI#2o<)@=)7kxs4wi;jimH!P=9d_9KV}at_*r}} zb4}zp`K^E+VkxwbDs%m|1zAoRS}2KzM@^kHy^yuKMM_5PM4w?&V>jC}*DBXCZ*O{< zyTlbnFJjwL``zbU^^TRcAHF=T@#)__6C8Xpy4BFy$BQvo z+J`&x9RnOqjv&Wy>oJqRvzMS-aC+!Cg)r<&+~L%^oPgrcmXlj`Y16s#cj3kC@7a%X z59M@EeH1xfRwp{avrt90Ew&T3=lFhg$3}OZrtX%qL_ic_Ym?A5p4~zCi%FztAsz+N$)@aEFS$2mG}@|F>&3r+%nZV z))wfDarJN>a1^*&x&mx-Ojj(O=ucvPP*YGz@X?60__wJWb2}HcYPq+~vo>wYp61WZ zt~hhFcJSpM;y3yiT4?XTNuNX0K8V|7;au!%5R zt$plj+jjDr?WJ?IQ*S37eOka>)&9F+W z-1wBJ^<~P>U)|)=Gl`$?e&b7yaMz|K9FHDt9l+Os88k>=rm)WoIdEvI) z_?*hT$weDmgcdn+BXaI#TuP{lx*O6hc%Q-}lL}q|C%iA6GDo~E$cz&4=21i>nu}7{ zTYS4I)H2tkBw(}4r@E}MgjrWv)2uhmZOmUy6Nw8%Yg0cW2497}z?P#w4Mo@x>?W#4 zYw)S~ZR{lemzZQaN#vUD`r_^;D$T1**+vpwZJcBa@Wj!{z<5r7{!UT6Y85n za?xbz9r?QO-Z5cupJTmoIf?%!{z_&t_Gjc}JWMB3s*`v4@?dzg`LXXKfw0Mn`7+vX zs5nQsiMN824J)8I)CaHIv(MAZ{nRzXwZ%TvKE-Y#S6J3tJ6HkB98(w56=SZEXWW4= zM{VdIbQh-6|1k_lM;d+_R%1LAG5o^!;bZa7#+JC-q%cn=x?1*|BFtgrcb_sC^W`d) zWrKT-J+qBK3c`&FtXqx0iM8e{gv;FCcAcDN`$4U?bYg3~ zBfSo`9(V|%yjk2~{N|#6M0w%~{&yuMvWft{EIhEK{8+%QAgQ8Ib~0pDU}t%4kT7Ur zK&PP70kJYDFwS4@!a z^TpfAx`aw1FWXl;W2`U9p5!CjR$_oF1G(WCZyIRYWlb}GH14*wMs>EA!~vp&bP#7P z2Ti-oZLHPSFZSjn>-5+-E|IH)=cv0kb%E*(j;H#tS9xQpxx$m+Ccz1Qn9#=SC%z?o zB0lPuBhHjwk-Gg}NlNAKh1TFY!NK4$qQwEL#Ipjo`&f^W@?x%ka97Sk*+oH15iZy# zXv2%*!NNpvuh0yf0AF*m?St8#kJt0RQH#7Yv8H`|abk=C<*l${pjEE}SP$5{!_R z3Lc5~NV{^^`Y#u(5GM22cNaSnqhcQ+pgS_Ed2&7uFC?c7OVYx7jzKvOaJ!=C1y>TYrlpeK_nsJEUd z@(TyY+i>oa+0<}niihX=L7jBi>29|9kd$l>MFF#|k>dB3I*u4<^sI(bIlU+kw+pDJ zyYfCmDbOp?UEnsX73>CX2s*;|1)-c;@o~;o;a<)#;RiTK(2P4<0P{?o4B;$hhV+Fy zSSs?V2`;k)e*ona44}tx35Lg;?oHu+q-vm}%vL%a7P)rwMmTQr8XX?6H@(s`7zS-q zc~SN@(0;0oGaNeOE%CenmChD`-t*d%Oq~L}j_2Tf?+(uhW-+6*ALOLD7t^cgKi*CB zC%V{iOEie+7QnIW<&lilHJM-OXfK}NItT9qYv~N$Iche49#tkBM9+dQ3aZ_9Q6E6X z&J^{fuEE22#q>3PXO2H}M)ZVAfb-x8pfkJ}tb*Ec=7R|U3hsu7P$J%XYBPKVIOp0V zN^+fm8>x2OmhNA?WXBP1ciNA>%r-*39HpYomd>I{+^nZUF@Qba4ny3m+gZC-)o9V%xgb=zM zn&?x!u7kJGN5MhtUb;JXwream#VdjCK%=QaoEhE$T(!3sn8;3HF?bn$fmMJkEd{N> zG6wVc;o1XH>{nXt31M+!FdSis99hHqXR#B)7hE450I7SdR_c3PANQrir`u3K1?@oD7cRv?%n9K zGmhuaq5FHkgBEH(aGgE}(`*-@5nN2`fiKJ@H^al}ARoRr8<6_kcNO3p>Jjt{Jn0SK zex&X&gIPEDj_yhK;QnN|49UC&-a!V+AFO93KoNTz*iHT5hCnyHC+GyA9k&A$LN&r# z&UnezHzfgm`%Q*}AN9kbCA8IJnhm-DxzyaLloQcjZ3=J&-(!AxMhj-f5mlbkv zvnA{!&pLPyudnAN9SwfuJc7D9Ch~(I4ST}-0~`td0B*P!vYFgC`ZzNl=m{9vbWb(( zA6N%H_bCS`uo=IxUIM{2Y^T4j0UBCc3o7aXu#=NB(AURy&*$0jRzi~KV3w9S14*dZhc*231++rWv z&;i`Rna?iq4g|l!J-rV10{%C6FPq_f4;+Je>||!WcPeKQcO1=d4g+D}U%SVbBrRWUF=}ab*ecC^IvfCfHCfFlnFe;SfTGsAthsfLu#;|TEgNC3}gYp&}+cW z7SR8BobX!E2DAs-FhahAA(&hAK6W)*z|4knIRqQ*>EbQtX7T)d=;d6dgU=EAld;mj z87rgmc}vg14}qz)nGJ<%!5&_UZo_m3_OO@eDp>+A-$7cBrbf;WMsz)f~K1Oi7rK5s5Pjx_`8I5XgV4C{GEou;xq9X(yx z5H^^uplBd~z3Tnt{o)o{ z2Hd{q34}huG;O(GG=rn%eByT%6!FjSE(#{|y7HQH-|^zOHcl?rr;g)^ z_^$*??rE-@(~Nr&?gTI6PV+e`KZ2#u9?%U=fet`QAG)`TedkkSY-jFxPg3E`DLRB+ z>do^`@CJK^x@WkAF4XzQdEFW7YII$8!tOrq%gzeNQG1fZY7cdmxqRidFVnnpK6kC~ z{B~JI#jI>`b* zjvp)eF6kHzhV9(0tU)EC`v;z!vqmOBG*LYF>!HdT8l8fGf>eR9obEq};M4!e659S9zPz5E<;jP$LnK;Bi+Kcsuuu!!AJ zL`=8%^UYr+E==B-QkOC|B{6woVnOqpaq^gnk@aD7LUI)uK8}5w|IGhuO@tQS8F(<1 z4oH{)s!umu6N8O1Y8`~;=V)NIDgHxWToy>TUsm)YoplP`&riA|S_oFi- zJRw1fIf3c2IBA@O5&q&Y=E{60auH+lesgnOYaAo(yM6BKg`}LkYt6G>vdAqH%y&!( z(aIP17~?;96^>);u_){{x)tq*CZn-F4{m>7|Naau#kOJ?7LSj^|HV-}(3oP(F;*EX zjNOgxjVe3_PxSeGyQ7z|n?x4*-WAHO=WY=DihM<<@FP)?v5E2bnkf>dCJsz~kXoE^ zF-wy(C+}9?{M_X1`{||0!Oh~M4~Gp`+zPNrt&&#aGr}0bMqX!lDR_{1?KQeuIQrOH zlUJ?dtk-=0&kN*oGM?OFU21WejuYpN@9`jC%!PP0-U)AoH}kC=1Pw(G8UhX1kt+z_ zaKz9bO~bxnZG7wgDt-w+j<3Nx<5uh~X29IoJlJS2f65Fmu$5G(}O!w&ZxhX;3ecZWj`x5GKyExPVIJK8hz)*k=gcX-oJW_Naa zTB^IMtIPUG*j4}KUVD{ST-(U&UAnlG%Id-MavrSTM&b>2+IGPD-kJsWgA?!oVj!^w zACG+mf648}b^23a3tXaa(Env9GX@w(7$)j*?HtWp^?vY0Dbn!T8Tu7q8y99SHwT-A z=1nFRa1}R-09Rj_{xtcRJ{eybZtDhU?2Y^CEp@*)PSUT$_VLpd?g1^D7-9}3=#wH+ zdNwUio0@SytFc+v=K2ChVbj8-<|sQUH94+DcvXPLD@=J@{#rIp+E3g~_>yPY3dW6^ zL%882_5dq$EW%0RD7l`jB0CVPup--a%VN`1LwAF};hXNJuD^bpuC>;xX`?xz`qcQQ z@m6D!Do)*2^FTXOzs<10*l7C4oNY0HTvB4|Y+Gx4Yr|~kY*uT!b(ST|Jj~cp|5@Xs zs;wX2P^QhdY1u1^<-zY_&L;(>2BwE+{GE9<=W_nLLQ(O#)`B*Dix(6G=Xf-g#NwfZ zpV706(%bc}bdG2Pe;Tc#3W#0!-(XoglvqpZ$)n_KY8u^(?nB?C4^!{(CD=CG@8+3? zsrqNSsoEEsWbGTxezlwWuxekUwIQ|fN8?LXAB`1SoorlfDmM?eJhUFOZMU~~EX5go zC7~xOh~>Bk{uvu&FSb1fFZ^=jWL+QCqWY`#Gt{Kzo8W;@Ni>x@4D0~kWVg=wm^-PU zuJw=hTRM*D@VrD^)HSOzense6-#{3%vyvsf2k)5ycjQFa;k zj@{2JqBfAn$Q7Uk>QA?(_Ta;9Cykr5i&bA5el)ymi~;|iV2x1StZ@o>q@1m*XlSo~ zrwcaCvfQ_xv$g}@Vy#tVuXn7Y8WV9nL9=e}L5 zDQ&FNUNzj&|4=`vzf-fWW@3H1=9qD>T|xNb`|bU0!!4^!BP{0~67X{WjVtGmu`>D< z-WAUz+5^TUgLGDc_p)v@b{ zE|x-UzvCFuNP955NFgy2xS0+xV=$r!aHc2z-Bt#DOE3g!%fR<1LZ8h%@p_UtqUd4i z{5CC{U(WuNBP`@fzjp1=<$0^58-Wke`5lM^xV_G*KNs?QIPufae4RR#J=Q&Y=SOPR5c_LOFvq0Dl? z-hhv$3ds9dy7i!Wp()=?Te~~P; zx(ZFSYO{7ZW%2$aIi+P~`%`T;=N--HnYF(ly#1e@f=d3(5`~Nq_Oo*Oe~rEDzH~94 zDA-Bo+GW;5*e+&_+$Z>9?DIG#Jj(l~q!)4}*O294|>H8Al+ck61yH^VkdjJ?5baGWHH@xeAXYj67-M_>Fs z(UIQBBv4xHi0!(4gzb|_VqRp+wNz+P!vJkPaoW=(X;sS^9d?xvIp3R}%A8-&uI;|k ztd`4@>>h9NcWP^6u+GywjkpPX{0r*1!+?Fn1BB;&)<*A2X_8`#X!Lv_8bnLTA|{<1 zhkoFrji~nAmkys-eeG0d)^D-Lk;T*`yt#de{T22Gk7U;JQeH>Dacr~hGsfz>=?|LX zt;?+&tO>R{>lACgEynRL=|eB3&eP=#O?<@GJAT8PVItc{>j&#GbBKPbDqQ7n`7RYl zEzG}Mnp1kPd11OTb5DM&;sGU}i=HN}^6AVDHdq_xY4oN@;xC~>?k0WD`?9aOyYi+X zO_G{q+)SGsv)TKecs+X@q+$IaVnMLkUG?DG$d8gwiQhu&PU-8hDP$GVh1fzQ5?W#m zN)*KNuFPR#fIZ2Ss#~GGsGF*9rt7UcYj|NyH=Z_&u;xSw)j+>s{MpMWmv*IEP+bW- zw#ELZ^@5o<0Y9VrtiGYTYwqu2jqOv=wDeNR!2-AJxmo$m*SEe}{3)whxYaeD+^N}F zGr6(e^gA(+6G+!cO8IxlLu&Q&iZ9K0mDwTXaudwMAodlwa5t$Q*mdL2#*g1CKMr_L ze74ti)@E5Q;XUXY`WpQM%@xj+ew6GKZ4}Pq9^)r11I(99&GgSS)mk_GY@Jm*-XOFl zWBu?XDj%hDCYDBB*oW*~<`J1oR5|Y1DlH)UZR@^;(^JZW%+ht%wY8z&kJampOUQ+S^J2B&FEpJ?RLH_RCzquUOFaf8 zIZN@I}J+RLg!^*KWe(A*^A>zFz0KkR6}g>by6l`xoljR;gpX2H7o)0SY` zq02SU23UEu29-^nOf2x}o0i{ZONSn9Kja&--!{A4VoTx4>`PIY$0Rga6Y%x;7ti`i z6Hd($ei4SVa*zw{@|zXw*7QKr#R>JHg!@owrDz~`k!WuVHm+{?{Y%07V;|doUscyy zhuey9-qDrX#^06%%Llo!iivIxaWH3N`jIu@g`Q>Vum4x`H~4KDwI0T?<^=2@d4_I8 z!}+}erRbt)pRlE%A7`Tj$U4Vk+aSx|z{4~*1_Cw@Gr1aHYIM4C)G6pz=Bw6RX;kYj zxob22%5L9$V_sU*`moK)Pvn)x9bYTHWj1!Too3gI2Z+b;F+3?h6|^>jk+l!w+)`n@Cn%>&i5-piQ+$@N4GeP-5?knr!(ni#Z zdg5q{b+lv}N;F04HrgHP1&yWpXV%lU1sFw*;ePNN_(0(!aipjpL|F<#Rls}x16B}i zq5aQ{8;p}pHdB(t0{qJ(W3Ho6QI#le`Kn!L$<#tDry%=u?%MpK?4Y#Cowj;G+ z-%iwbGH;PtDK3Q{ye7VE2DqHrKV6%N_9{VJu4%v+OU-V+^6XkLuN`UFwnAO8p|^ zQtN$3DfyTV21(mJfhRdBR}*XUZj+);*|wB(s$OdE_B42zBP%?8URYl!_%M=y*u z->`R+8{*W3-%H!I|4?kpYun74Kf6GiJ0#^qXsA-pw9&tX_4=#1B}$Vp@(|Zb*(5pR z-J!|Xq}OR>O;;vlMdbM0lP8K*%oInMWw+){&F8P#pSpdj{&uBKtMM>Bvvnt4FtOZk z!pD-g(oFGXVSwN~9pX?~SDM~xW$Mq3IgO@<7mX`40}Z>(t8J_CyL1-&fW5>|75WLf zum$Wj^n!ej71=M_qRrKYEyiZ%NYeo0Aj=n5A#F0h0Q1|QtYkPiL$l~m|#V6V| zv^mgxP44Ly_9CYFnyeA=cLEz+`Vm77CfzVw21xH5t}R{9%SFoJKAl4SbI(fXrGwg*wcw8^c8LkFXZn*jH^qe z+>v0Dn}%z@HvUu3*L13JtLs%yG(OkvF}yMRI35$hln-0P8`*X2N%k`%r3T@5uy*z* z<|h#KWsxz@aNc+ayk2%#_h1+fUYMkT#+a$pdF%r=N)#6wnsv9}P(fz%`*~scTMB|( z{aN@U=UUSJ&;~^VdkG&!>bL{qJlQ0W)Qpura0~I>78(*eJaKdUjOag`6b6U+weY&< z7Asj`;+9WmBTRy|?9}CEt=_Is|yJ z)ly(9#xRE;QBI9!Ow4U!v{WT#lF)eON)U~j+Av=PO`aE>25~YwE zj91lv*N4|=e@yyW_G7`f${$QE**H_T-`v+e4LgrTJ0>_LVe{>$ZO_bm^gA@IR3q!B z)%{yv(6G57M!iRuYA7@HvQHvsrAQ>x8m*KK5;$wn5VYLYCGCg90M?{^BdmQT*@jAFF=tbaKzf3QK>uo^^ z5n}PvC#jdI&Z|eL%IeNTRFdCn?e!m3@wyp?cLq#P>+c&fjWbL(v%fjdupKOAN+Dvz zS}m^g(|0y3FdB^QEgx)iu*3Kw%7d0t^XN{@AUc40O$E^L{*76|Ao?s*!YrU~Qw8*OlEjp@2FF3!(s1|GPZ_~kUU`{?mR4QbzAo8a zQrdQN!IQ)`zIJ)J>kY*?HzXS=ZYFFiOcExFHp+ecR);+a{o=phYm@sLWwrZRrLR0q zbcB0L9yMRrN_DLbBXp0{e>C*2Ire?l50~0bwU)YK6{p^%vNuL+Ty&v^Qqv%Fx~Zjc zh;b+E*xwpknva=rhyWF8-DjPO72>5}6}Oq{PJJi3k=?2By{JEE3G)}NrY|#nnK|?&I)%|thsc%GCejz{YH=kp<%ZCIl3J#r^ugJ! z3LX_3S`TeoQ1-o(tiSg{BwHp?aEZ9g+RNfKQVt|5xJZ6qyp(JbTXYsYpF%lII4j9n`}X* zkPBc}WF<1l^TZV*lUh$Hsp-^0`T&ffncl#V^lTce`Iuw$P3i_!Mt7v%Ir6a6)Fzj( zfbC6=MumcBxqDhhrdw9u>^pgNE%xPIO<5N~1@{Ok4`~*l_g>*S!^5B$FaIPxAU!1y zRK_b0$xEbnCHEv|agpeuAeGN&uhN@|7LNJYO6-h1+P1^e(d=%@G!h1pu@XGn-x}41 za>GZ%XXEdtzfGgfO3O^kB@1WiVZClW2=VcOB#Au3X}Nz@yr8zSn&3fj)`84}7Ej2KnvwJMA|X{Eid-+WL9=v+IKS7_Ns>wyfdHl1(AEal# z*ldvZp0LfZ^#*xVwQaD?$L0lg57oApwgQj|zqX687Fb_wBK8>@?fBrx#`oYf-i(+* zye3+ZXUKGFGxdZrP#N?%dOwZPy_g-$W#%PgXL!a7WuZ#+2ilDepe{&^dI9e`5PG_r zcN2UP)CmcZM*LACl8tjYCSUB@)Xm~XD{|a>dn9|F^Gx%)>y_dChxcLc2i^wnM4zrc z;XY@)OT6EEb@saJS>$=v!_#A!`)1{4#Td6St~2GsU7}^)($&znh8r6&1NvRc2kF$@n-?zJB&9U*=5^N9l z7WRFotCkktYg3lh8{9vSs!RyNg}T zo?s8Mo?H|+ob%(i@F#eRXZclpj9`m!o@j!I6uL_l@^SKWE@3V0JdmW#NZq!_=BPAj#aUp$*8O=n&e3?vVFcx|tJq2Chf&}}y z_e^&>j@`>eqMmGTz6IExWO9LQ09q~Z5UuA4G#f4De;2AmYq$aQWUjXe7tZ9yaks=% zl#L!2+>G)y$^|~#111Gk2ikl;`JD)~2X_wQf)9nA3$q6=3-}`-EMRbeS6~~zeqJlw z|90P_Fv%~v`~%QRmZ$`S_$S=&Tm`?9yM;P1LrJ}3nxnm=*m0S-j9;+MHMTXr1H2q$ z*=ZhP+GFUebJzXQ_Ax3fw@g)rG+nB0i>bZ!j{UZ6x~01%-M+|PZ@-0mQE#YhvI!oG z=a8T1&u9edNKOGt^q4Zzg*2jamJ zuwc1`TzMV;l3fDpPKuzf{IpW5=;3xt`Ipbl;LylTF_&WR$DNDMNqm`nG%X}6E4ydL z)5Nlvy>UB|A`@3d$AlUD`g*={AK>nyK(Z?F3c*Zf7!gmdXY@=AbpY>zg%X)myc{>Gv8g7^BS5 z)+hF2yd4os=FkbKHG7NQ&mLvpfplssto_T8g{h)HQ==#?u?2sPlQ`iR>8QZ#@nygh z?Iqum&8e15f7G7=`o-zE1a>EDTkE1t!EaQ*7N(CfYOx`#2aEG|Ano_#lc zS(lV}w zs&T2Us-01{tMQwryH0MXGqtwvu+Jnm(-lk*y2!qd=;)0RC0qM2~ib2h~7ym z@HdVSd^xt-9*;?}5%#{=9*5bn7WX18gx1)#&%|IDDz5-k6q{(O+2RBQLRB!SqAHi-BAPDv~_0Opf{?GWA#0?5aUPPT^*xw zXy$3%benY@3@c5iEu(B5F~;$fsHAG?-_UUO8`}vq;WN2v(Af?|BY{SnsrTe4%9H%; zNWs718IC#FGobG}tijQf$R+j>&8RKZ9;yj*1lFr+q-SG5-`t$@;H_*9`-6Rt1~d1W zdMd)tCC$fXT*w@fzof;nX0EC8 zqXz|V4CoX%$hWUsqOc9M*#3{L52mq>*3VQ`HP)#vYWo|a3_Wx!bpwnGfzSPH|JT0S zz7+GtU5HQkYW%(9BQ^&!V)57ko5L!wDXi@*_bf5i6ic?{owW))3i{gpArgklF@ZQl zHjoT(91eOG(+cSJ8KxM$V!F^;+Mj7p8HrBRF0v&ag)b+bIND<`9b52nY!Y?@Y{JtV zIKG^mN8Ew^&~;i$H)AfLe&`4?uo2v9wua5+K0w@P0lOWwU_!}2(jrK6Pxn0Sazm2m z;S(N^n4Hu(I=o3^yfAN9tB);OWo%6unmM+hq9`!uQbM1oFHuFY)1!U_ymp%`AwIwcoa%#8^i)ew%Qiwvy3= zn#d<bwY(` z7ODeYc^gyCl!4xTC$*gnAiI+d#2Nf2-j~>qAH-{ks|10!CojX^yA61!oM+ZUbRSX{jSET(UaSNPJX9N!9oXzk(Y3Gqh*F@Z@_lG;ZF`wv&Bjt=$}y%iBuqS5P7e25SDKZKDwK^v*7^mOJV{gS#szo9o$O{lX}Gio|$oV=<1 z9NBPkmX&<_r{!UlIy3&_kLRV9WJPCWMP|`qbgh)Hj@MK~2 zTM<5>JE(x@9{%KPay!+Q9!a|}I%W=1p=Im~HVP=?bM7stV0q)#$o^aN@+olgHGM^c07rIdjT zq8HH@DPP)``I}CGF`mvG2F|@B^9i`x>GVwc5LF5u85XjTJV7obzYtpi^UTB{qCeaZ zA?K1br38p?0XzQl!0Sh{8zHXqMD`Ka58~3F;}|ZIs|WkFC+s(r%r-$)%qIE-Wdm!{ zqk`+=YrK|C6faYlAa2@fp9@|WgOcLYBPD-5r|NWrH)BEktm3) z362gL6s!*z9Gsj-|TVKNw%>4f>h-^%`B{l!oRICHI25L488~P9xTfP3z1X zEcdJlSTp<$;YW3+f6@n-FK8}X!6vis*vV`UwjJAsZHa;q*oiUCfG#Ld4Sk-T#}qU7 z=vSbFc}IT+4cJ?HD}4_3pXVqST2HxCS4e*f^s#^+Mxq~4Pl!lwB7!&x`s6WS?ei}+ zni5f+=xEqsHDQLLX{Z}&M1Qk?vqwP262w(-x!hZJHhY7e#dcsvBN?h>HZoV}(_{$| zLEUEO@dD5gh46c&Zc4^|q_T@L(?=V6DC$mRWD`0fD`98a(u_7uPbU;4m8A{In3~cj zRu*|9;%P)h*e?GhkI$~*aze@qcF_+UKK2@Ge@mXRrFN=1OLJONtv(6SQqQU-+D6?% z{dfIUW2t4AwZE+td*}EE|4HTBA*q_1ATB{X^{o&1%hM zja1uFw_ATt|K4!N6lS??*vH9F+ zZUA^bwB(M0EqgauAs(YTM4_F~&z@*M__|zYHZX1JFnToo7nM(4q_$p~ zLOmyc5*o6e+D%nb>*y!+4tg*1FY1kwQ3qDSU1EKK6DxpI4c>4Gd?eSv&H``QJa#&8 zrt=U1YurWV0PVxfVLni?WGTRV8~qqCZz$6Xg$k!hpSUcN`?%&S+IZgzTpU~-)H3)* zcu~ysgo=b+v7*@H35H~p)GU5a^u@?OB6A}rgmnto;r-2Xp8Flwh0>P7^QbSSA}~j- zt+`oge5+42JkyWRZPeH_V{|9=N<+F~hauXu&Y}d12(xV~HW(jCRFXRK6je>{XA)pH z_%BL_$nOT0XAwJ(8^STHf;-7g=Pt0f*#=h2c40@e-_Z*+4-G+5bb{H+e4#G@6`Dn3 z0H@Kg61Airz$$lx+y<~(N_D0PpxVB49W@4arsYf%;PCgN%@ASU0=t?&*{xi2{tUgYwrrVf zvCDHePwzMWBLmX>M+Hs|TM?BKTM$za)j7Ib{P9F}e5aVJk;#!~Bl9EPhphBp=u_u4 z%45BoO6n&1z}-YM=>)uKCm3x8VB3P+$dL#J!oie^R-88Aq&#XMyV=Tc| zIR?O9qd8?FjpSaMV!kj3ffmg{O2C3#wjK>-8K4zS**n0^$HTolyO=G9xwjo1XKI;B zwj(CQAT!w~|me--M0Ca@>DKrR5H zdtGBkb6WsYgsc`g*-Wl4yBckU@y|vf=q3}!L^FjTNiL*p;N<}LLyacu$rf}rvyQd_ zZg!*7(E}96w4xu-er#vXkK<4*8qJpRLj+aaU^bb%$43j=@vFJ_tdKYIbNOnvHR})d z_l<04HU#a4RviIL<-3#+C|_5Q`Qhk0(~o^G=ph*(-5@QHy>T7pG2VNgcXuD&Z%xq9 zP?vCV_+Md*n;0T{M=gsO*JNk-;P6dhw?dSG1N?gWp7fsKvD0mr%V)_Au~xW(n?&CM zI=h9q?r3A5XDzcXw(YgOv&b!LAqwI|+ZWq$o6H`86*#^)Jn-9iC&HC{4LCH1s-?C8 zcDXQB%md~#69JN|erOjGvL5Uy^bEzZoxqQfLyLjhUq>$N8g!l+$_#?N&r>F!p@3_g zK)W()Knfd0ErXS{lIDTm+63IRED_MDo?tv_lodi{r&xu{i6Ig1WJN;1Z@k*^PlQJDqw?ucVBPtO>*%p z^4PC5xHhTt0cEgHO1z1@~wWorqAgY3jrrQJeY^J)=FX(*GF7yS?p^(8q0_X}Yt%ddQ2dr$t zh-5Cn(-F){uwvfIWI|2ROnYWNvzaM^Z?l=e-O^JvGCt{u^?7}^BA0!!|*u&m>&*$ zbPH`nRp>UXle<7Vz6xM62%SJ{07oaHJ@Di$Qm{>6-{}J}8aA?ZKy_+>#t-H$!7RGT z9R#V&DE=rvi(dfOe*p)B1W9mq({b}BN-)?%KT+VWOmsJmlFAJ@)&tL`FlAje=Z*_cafiV`Pb!^OR`Ie z?1glWG*kKvd|)cX7vR*PVIqSt7p&XM1yKS!zn4$uYq)!G8c#Q_8Rx-$0t$W-q+|ow zKA`i>23{(j6$3~701^R7#_k6U`;Ic%Owa_70Ndt}|2A+BPTq7ddyy5u3K<0b-^{9D zT^q|rK)%(GV<+1dN*ur*2CIte@N^UKObXx+a^b(2n*ry))Phz1JI=x7f{(xgkX-fV z5A#Anh~PWFo4?D43Od90j{v7IA&}-uze#JQ1K<=QJ)FcO0Q>n&$tCejafP^z_@1b<=%w(4 za0d7#HV78O=~xJ)qJM)Nya=Rc8txf)fjbXl*BeeP%HTS2ML-AS;N_zR?!b>TgXHE7 z@C0J;mT>1BEXg+E`arLeU&mqS?%@(QpXdSmF_U( z;`k}Hv`rj zN8WV4YwI%(UMH zLjbnLg64uODA!-$Jd=&!4ZIWbANUviL%`l^{7wD>oV|98Kf><^dUy!1Y$tFk+u(W- zuFkX%zMTL#p5V{$7a{)*=*><39{&i^GbraJq}Nd5XGlNzS}6S+^s@%iPdJyXo^OO2 zEs)GSI9|e-GjJs#A(#`)fA>Owh6}(%B2WVL@D(T^DWSa{aCH;7LQ=q;s{kBfcpmVK zaX(uSC4Z}IqP%2_5a7;{{tWdN5B98 literal 0 HcmV?d00001 diff --git a/ci/model-artifacts/manifests/competitive-benchmark.json b/ci/model-artifacts/manifests/competitive-benchmark.json index 3504c73a95..9a70a8f56a 100644 --- a/ci/model-artifacts/manifests/competitive-benchmark.json +++ b/ci/model-artifacts/manifests/competitive-benchmark.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "competitive-benchmark", - "registry_sha256": "0fb272556fa31f2ab91d06c8265af82aa531c2001dbcbe6c3af3dab941140b3c", + "registry_sha256": "7391ea78891c52d41a39d77a2d6fe18b9630946cdbf28009f6a761ebb18a3bc4", "artifacts": [ { "id": "family-llama", diff --git a/ci/model-artifacts/manifests/hf-download-smoke.json b/ci/model-artifacts/manifests/hf-download-smoke.json index 05aee70ae5..0d04439c2b 100644 --- a/ci/model-artifacts/manifests/hf-download-smoke.json +++ b/ci/model-artifacts/manifests/hf-download-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "hf-download-smoke", - "registry_sha256": "0fb272556fa31f2ab91d06c8265af82aa531c2001dbcbe6c3af3dab941140b3c", + "registry_sha256": "7391ea78891c52d41a39d77a2d6fe18b9630946cdbf28009f6a761ebb18a3bc4", "artifacts": [ { "id": "smollm2-q4-download", diff --git a/ci/model-artifacts/manifests/openai-smoke.json b/ci/model-artifacts/manifests/openai-smoke.json index 9c3ccdf388..45a7d016db 100644 --- a/ci/model-artifacts/manifests/openai-smoke.json +++ b/ci/model-artifacts/manifests/openai-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "openai-smoke", - "registry_sha256": "0fb272556fa31f2ab91d06c8265af82aa531c2001dbcbe6c3af3dab941140b3c", + "registry_sha256": "7391ea78891c52d41a39d77a2d6fe18b9630946cdbf28009f6a761ebb18a3bc4", "artifacts": [ { "id": "smollm2-q4-download", diff --git a/ci/model-artifacts/manifests/product-integration-smoke.json b/ci/model-artifacts/manifests/product-integration-smoke.json index fe015820ed..c8f426210f 100644 --- a/ci/model-artifacts/manifests/product-integration-smoke.json +++ b/ci/model-artifacts/manifests/product-integration-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "product-integration-smoke", - "registry_sha256": "0fb272556fa31f2ab91d06c8265af82aa531c2001dbcbe6c3af3dab941140b3c", + "registry_sha256": "7391ea78891c52d41a39d77a2d6fe18b9630946cdbf28009f6a761ebb18a3bc4", "artifacts": [ { "id": "family-granite-hybrid", diff --git a/ci/model-artifacts/manifests/product-smoke.json b/ci/model-artifacts/manifests/product-smoke.json index 30347719d1..05f1778a8c 100644 --- a/ci/model-artifacts/manifests/product-smoke.json +++ b/ci/model-artifacts/manifests/product-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "product-smoke", - "registry_sha256": "0fb272556fa31f2ab91d06c8265af82aa531c2001dbcbe6c3af3dab941140b3c", + "registry_sha256": "7391ea78891c52d41a39d77a2d6fe18b9630946cdbf28009f6a761ebb18a3bc4", "artifacts": [ { "id": "smollm2-q8-inference", diff --git a/ci/model-artifacts/manifests/radix-cache.json b/ci/model-artifacts/manifests/radix-cache.json index 8cedbcdb42..21c7692c4d 100644 --- a/ci/model-artifacts/manifests/radix-cache.json +++ b/ci/model-artifacts/manifests/radix-cache.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "radix-cache", - "registry_sha256": "0fb272556fa31f2ab91d06c8265af82aa531c2001dbcbe6c3af3dab941140b3c", + "registry_sha256": "7391ea78891c52d41a39d77a2d6fe18b9630946cdbf28009f6a761ebb18a3bc4", "artifacts": [ { "id": "family-deepseek2", diff --git a/ci/model-artifacts/manifests/safetensors-runtime-smoke.json b/ci/model-artifacts/manifests/safetensors-runtime-smoke.json index f333747029..f548dee6d3 100644 --- a/ci/model-artifacts/manifests/safetensors-runtime-smoke.json +++ b/ci/model-artifacts/manifests/safetensors-runtime-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "safetensors-runtime-smoke", - "registry_sha256": "0fb272556fa31f2ab91d06c8265af82aa531c2001dbcbe6c3af3dab941140b3c", + "registry_sha256": "7391ea78891c52d41a39d77a2d6fe18b9630946cdbf28009f6a761ebb18a3bc4", "artifacts": [ { "id": "smollm2-safetensors", diff --git a/ci/model-artifacts/manifests/scripted-binary-smoke.json b/ci/model-artifacts/manifests/scripted-binary-smoke.json index e62ddf0cdd..2b2dcea79f 100644 --- a/ci/model-artifacts/manifests/scripted-binary-smoke.json +++ b/ci/model-artifacts/manifests/scripted-binary-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "scripted-binary-smoke", - "registry_sha256": "0fb272556fa31f2ab91d06c8265af82aa531c2001dbcbe6c3af3dab941140b3c", + "registry_sha256": "7391ea78891c52d41a39d77a2d6fe18b9630946cdbf28009f6a761ebb18a3bc4", "artifacts": [ { "id": "smollm2-q8-inference", diff --git a/ci/model-artifacts/manifests/sdk-smoke.json b/ci/model-artifacts/manifests/sdk-smoke.json index c55578838d..53f8fcf782 100644 --- a/ci/model-artifacts/manifests/sdk-smoke.json +++ b/ci/model-artifacts/manifests/sdk-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "sdk-smoke", - "registry_sha256": "0fb272556fa31f2ab91d06c8265af82aa531c2001dbcbe6c3af3dab941140b3c", + "registry_sha256": "7391ea78891c52d41a39d77a2d6fe18b9630946cdbf28009f6a761ebb18a3bc4", "artifacts": [ { "id": "smollm2-q8-inference", diff --git a/ci/model-artifacts/manifests/skippy-ci-smoke.json b/ci/model-artifacts/manifests/skippy-ci-smoke.json index 560e2b7989..af5b01c25f 100644 --- a/ci/model-artifacts/manifests/skippy-ci-smoke.json +++ b/ci/model-artifacts/manifests/skippy-ci-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "skippy-ci-smoke", - "registry_sha256": "0fb272556fa31f2ab91d06c8265af82aa531c2001dbcbe6c3af3dab941140b3c", + "registry_sha256": "7391ea78891c52d41a39d77a2d6fe18b9630946cdbf28009f6a761ebb18a3bc4", "artifacts": [ { "id": "family-qwen3-dense", diff --git a/ci/model-artifacts/manifests/skippy-correctness.json b/ci/model-artifacts/manifests/skippy-correctness.json index 267a98421c..c3cf6b030f 100644 --- a/ci/model-artifacts/manifests/skippy-correctness.json +++ b/ci/model-artifacts/manifests/skippy-correctness.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "skippy-correctness", - "registry_sha256": "0fb272556fa31f2ab91d06c8265af82aa531c2001dbcbe6c3af3dab941140b3c", + "registry_sha256": "7391ea78891c52d41a39d77a2d6fe18b9630946cdbf28009f6a761ebb18a3bc4", "artifacts": [ { "id": "qwen3-q8-correctness", diff --git a/ci/model-artifacts/manifests/skippy-parity.json b/ci/model-artifacts/manifests/skippy-parity.json index 0e76600138..8222387f94 100644 --- a/ci/model-artifacts/manifests/skippy-parity.json +++ b/ci/model-artifacts/manifests/skippy-parity.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "skippy-parity", - "registry_sha256": "0fb272556fa31f2ab91d06c8265af82aa531c2001dbcbe6c3af3dab941140b3c", + "registry_sha256": "7391ea78891c52d41a39d77a2d6fe18b9630946cdbf28009f6a761ebb18a3bc4", "artifacts": [ { "id": "family-deepseek2", diff --git a/ci/model-artifacts/registry.json b/ci/model-artifacts/registry.json index df92a39206..8a8e2a1610 100644 --- a/ci/model-artifacts/registry.json +++ b/ci/model-artifacts/registry.json @@ -53,6 +53,21 @@ "tensor-ownership", "stage-load" ] + }, + "workload-smoke": { + "status": "provisional", + "oracle": "none", + "required_lanes": [ + "class-specific-smoke" + ] + }, + "workload-oracle": { + "status": "certified", + "oracle": "local-monolithic", + "required_lanes": [ + "class-specific-smoke", + "class-specific-oracle" + ] } }, "cadences": [ @@ -108,6 +123,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "cadences": [ "llama-bump", @@ -160,6 +176,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "cadences": [ "llama-bump", @@ -208,6 +225,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 47, @@ -251,6 +269,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 40, @@ -301,6 +320,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 46, @@ -348,6 +368,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "cadences": [ "llama-bump", @@ -396,6 +417,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 24, @@ -439,6 +461,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 48, @@ -482,6 +505,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 24, @@ -532,6 +556,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "cadences": [ "llama-bump", @@ -581,6 +606,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 28, @@ -626,6 +652,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 27, @@ -672,6 +699,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 48, @@ -727,6 +755,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 48, @@ -774,6 +803,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 52, @@ -818,6 +848,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 24, @@ -861,6 +892,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 64, @@ -904,6 +936,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 24, @@ -947,6 +980,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 32, @@ -994,6 +1028,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "cadences": [ "llama-bump", @@ -1042,6 +1077,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "cadences": [ "llama-bump", @@ -1089,6 +1125,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 40, @@ -1133,6 +1170,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 16, @@ -1179,6 +1217,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 16, @@ -1234,6 +1273,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 40, @@ -1278,6 +1318,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 28, @@ -1334,6 +1375,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 36, @@ -1390,6 +1432,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 26, @@ -1433,6 +1476,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 32, @@ -1476,6 +1520,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 40, @@ -1519,6 +1564,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 32, @@ -1562,6 +1608,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 24, @@ -1605,6 +1652,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 40, @@ -1648,6 +1696,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 32, @@ -1691,6 +1740,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 36, @@ -1884,6 +1934,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 24, @@ -1927,6 +1978,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 24, @@ -1970,6 +2022,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 18, @@ -2013,6 +2066,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 26, @@ -2056,6 +2110,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 32, @@ -2099,6 +2154,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 30, @@ -2142,6 +2198,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 24, @@ -2185,6 +2242,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 16, @@ -2228,6 +2286,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 30, @@ -2271,6 +2330,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 62, @@ -2314,6 +2374,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 32, @@ -2357,6 +2418,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 36, @@ -2400,6 +2462,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 32, @@ -2443,6 +2506,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 32, @@ -2486,6 +2550,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 40, @@ -2542,7 +2607,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 15823053440 }, - "notes": "downloaded AFMoE architecture coverage" + "notes": "downloaded AFMoE architecture coverage", + "class": "causal_generation" } }, { @@ -2585,7 +2651,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 5117561504 }, - "notes": "ARWKV recurrent architecture coverage" + "notes": "ARWKV recurrent architecture coverage", + "class": "causal_generation" } }, { @@ -2630,7 +2697,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 15803475200 }, - "notes": "hybrid three-recurrent-to-one-attention boundary sweep" + "notes": "hybrid three-recurrent-to-one-attention boundary sweep", + "class": "causal_generation" } }, { @@ -2673,7 +2741,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 4274466400 }, - "notes": "downloaded Chameleon architecture coverage" + "notes": "downloaded Chameleon architecture coverage", + "class": "causal_generation" } }, { @@ -2717,7 +2786,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 16131110560 }, - "notes": "compact four-layer DeepSeek 3.2 architecture probe" + "notes": "compact four-layer DeepSeek 3.2 architecture probe", + "class": "causal_generation" } }, { @@ -2760,7 +2830,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 10896057440 }, - "notes": "DeepSeek V4 Flash dflash path with four hyper-connection streams" + "notes": "DeepSeek V4 Flash dflash path with four hyper-connection streams", + "class": "causal_generation" } }, { @@ -2803,7 +2874,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 14302555328 }, - "notes": "downloaded dots1 MoE architecture coverage" + "notes": "downloaded dots1 MoE architecture coverage", + "class": "causal_generation" } }, { @@ -2846,7 +2918,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 240645504 }, - "notes": "small ERNIE 4.5 dense architecture probe" + "notes": "small ERNIE 4.5 dense architecture probe", + "class": "causal_generation" } }, { @@ -2889,7 +2962,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 13245833216 }, - "notes": "downloaded ERNIE 4.5 MoE architecture coverage" + "notes": "downloaded ERNIE 4.5 MoE architecture coverage", + "class": "causal_generation" } }, { @@ -2932,7 +3006,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 4237063328 }, - "notes": "text-only Gemma 3n architecture coverage" + "notes": "text-only Gemma 3n architecture coverage", + "class": "causal_generation" } }, { @@ -2987,7 +3062,8 @@ "sha256": "debad39ab9c1152ab67695a674fb35e8375b2320c57bfd5075835d3ccb16c7db" } ] - } + }, + "class": "causal_generation" } }, { @@ -3030,7 +3106,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 251938176 }, - "notes": "compact Gemma 4 assistant architecture probe" + "notes": "compact Gemma 4 assistant architecture probe", + "class": "causal_generation" } }, { @@ -3073,7 +3150,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 13135032320 }, - "notes": "downloaded GPT-NeoX architecture coverage" + "notes": "downloaded GPT-NeoX architecture coverage", + "class": "causal_generation" } }, { @@ -3117,7 +3195,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 6047918368 }, - "notes": "Granite sliding-window MoE architecture coverage" + "notes": "Granite sliding-window MoE architecture coverage", + "class": "causal_generation" } }, { @@ -3160,7 +3239,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 821847360 }, - "notes": "downloaded Granite MoE architecture coverage" + "notes": "downloaded Granite MoE architecture coverage", + "class": "causal_generation" } }, { @@ -3203,7 +3283,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 8428497696 }, - "notes": "Granite Switch adapter-router architecture coverage" + "notes": "Granite Switch adapter-router architecture coverage", + "class": "causal_generation" } }, { @@ -3246,7 +3327,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 20167310304 }, - "notes": "downloaded GroveMoE architecture coverage" + "notes": "downloaded GroveMoE architecture coverage", + "class": "causal_generation" } }, { @@ -3289,7 +3371,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 1133084864 }, - "notes": "downloaded Hunyuan dense architecture coverage" + "notes": "downloaded Hunyuan dense architecture coverage", + "class": "causal_generation" } }, { @@ -3332,7 +3415,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 18236703520 }, - "notes": "downloaded Tencent HY v3 MoE architecture coverage" + "notes": "downloaded Tencent HY v3 MoE architecture coverage", + "class": "causal_generation" } }, { @@ -3375,7 +3459,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 8967115424 }, - "notes": "downloaded Jais architecture coverage" + "notes": "downloaded Jais architecture coverage", + "class": "causal_generation" } }, { @@ -3418,7 +3503,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 5104022560 }, - "notes": "gated Jais 2 artifact with locally verified immutable digest" + "notes": "gated Jais 2 artifact with locally verified immutable digest", + "class": "causal_generation" } }, { @@ -3463,7 +3549,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 784318432 }, - "notes": "compact hybrid three-recurrent-to-one-attention boundary sweep" + "notes": "compact hybrid three-recurrent-to-one-attention boundary sweep", + "class": "causal_generation" } }, { @@ -3509,7 +3596,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 5138197824 }, - "notes": "LFM2 MoE hybrid recurrent boundary sweep" + "notes": "LFM2 MoE hybrid recurrent boundary sweep", + "class": "causal_generation" } }, { @@ -3552,7 +3640,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 672108288 }, - "notes": "downloaded Maincoder architecture coverage" + "notes": "downloaded Maincoder architecture coverage", + "class": "causal_generation" } }, { @@ -3596,7 +3685,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 8071294784 }, - "notes": "Mellum MoE sliding-window architecture coverage" + "notes": "Mellum MoE sliding-window architecture coverage", + "class": "causal_generation" } }, { @@ -3639,7 +3729,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 4965526048 }, - "notes": "downloaded MiniCPM 4.1 architecture coverage" + "notes": "downloaded MiniCPM 4.1 architecture coverage", + "class": "causal_generation" } }, { @@ -3695,7 +3786,8 @@ "sha256": "c2e08b8bd13ca60b59ffdcb79b969b354b1edda9b9198047d10621adc11591bb" } ] - } + }, + "class": "causal_generation" } }, { @@ -3912,6 +4004,345 @@ "MXFP4_MOE" ], "notes": "SafeTensors direct-load fixture used across every supported load-time quantization." + }, + { + "id": "family-nomic-bert-embedding", + "family": "nomic-bert-embedding", + "suites": [ + "llama-family-certification" + ], + "cadences": [ + "manual-full" + ], + "capability_tags": [ + "gguf", + "embedding" + ], + "artifact": { + "repo": "nomic-ai/nomic-embed-text-v1.5-GGUF", + "revision": "0188c9bf409793f810680a5a431e7b899c46104c", + "selector": "Q8_0", + "files": [ + { + "path": "nomic-embed-text-v1.5.Q8_0.gguf", + "size_bytes": 146146432, + "sha256": "3e24342164b3d94991ba9692fdc0dd08e3fd7362e0aacc396a9a5c54a544c3b7" + } + ] + }, + "certification": { + "class": "embedding", + "profile": "workload-oracle", + "cadences": [ + "manual-full" + ], + "execution": { + "trunk_layers": 12, + "mtp_layers": 0, + "activation_width": 768, + "boundary_sweep_period": 0, + "speculative_policy": "disabled" + }, + "resources": { + "runner_role": "family-certify", + "cache_policy": "immutable-local", + "estimated_model_bytes": 145389792 + }, + "notes": "Local full-model embedding, HTTP and official SDK smoke, and pinned CPU monolithic batch/single-vector parity; staged tensor filtering is intentionally unsupported", + "evidence": { + "fixture": "scripts/workload_fixtures.py#EMBEDDING_INPUTS", + "comparison": "batch and individual vectors within 1e-4 coordinate error and 0.99999 cosine" + } + } + }, + { + "id": "family-jina-bert-v2-rerank", + "family": "jina-bert-v2-rerank", + "suites": [ + "llama-family-certification" + ], + "cadences": [ + "manual-full" + ], + "capability_tags": [ + "gguf", + "rerank" + ], + "artifact": { + "repo": "ggml-org/jina-reranker-v1-turbo-en-GGUF", + "revision": "607d8664c787e517e5d6e339d21f680f9002c931", + "selector": "F16", + "files": [ + { + "path": "Jina-Bert-Implementation-38M-F16.gguf", + "size_bytes": 76971168, + "sha256": "71abc010bb3dce97812ee971509a5cb6ff6f6b8cfffd8480129242f605521fca" + } + ] + }, + "certification": { + "class": "rerank", + "profile": "workload-oracle", + "cadences": [ + "manual-full" + ], + "execution": { + "trunk_layers": 6, + "mtp_layers": 0, + "activation_width": 384, + "boundary_sweep_period": 0, + "speculative_policy": "disabled" + }, + "resources": { + "runner_role": "family-certify", + "cache_policy": "immutable-local", + "estimated_model_bytes": 75293188 + }, + "notes": "Local full-model cross-encoder rerank and HTTP smoke with pinned CPU monolithic score/order parity; staged tensor filtering is intentionally unsupported", + "evidence": { + "fixture": "scripts/workload_fixtures.py#RERANK_DOCUMENTS", + "comparison": "scores within 1e-4 and identical document order" + } + } + }, + { + "id": "family-t5-encoder-decoder", + "family": "t5-encoder-decoder", + "suites": [ + "llama-family-certification" + ], + "cadences": [ + "manual-full" + ], + "capability_tags": [ + "gguf", + "encoder_decoder" + ], + "artifact": { + "repo": "Felladrin/gguf-flan-t5-small", + "revision": "d71c51f67519edd3154527c2d8f20288bdde9705", + "selector": "Q8_0", + "files": [ + { + "path": "flan-t5-small.Q8_0.gguf", + "size_bytes": 113709824, + "sha256": "f7f769c360b1ba830b10dd3b7e7d146dbcc4d487962be7dd806d7d52e0a9c2f0" + } + ] + }, + "certification": { + "class": "encoder_decoder", + "profile": "workload-oracle", + "cadences": [ + "manual-full" + ], + "execution": { + "trunk_layers": 8, + "mtp_layers": 0, + "activation_width": 512, + "boundary_sweep_period": 0, + "speculative_policy": "disabled" + }, + "resources": { + "runner_role": "family-certify", + "cache_policy": "immutable-local", + "estimated_model_bytes": 112678400 + }, + "notes": "Local full-model encoder-decoder generation and HTTP smoke with pinned CPU monolithic completion text parity; cross-attention staging is intentionally unsupported", + "evidence": { + "fixture": "scripts/workload_fixtures.py#ENCODER_DECODER_PROMPT", + "comparison": "identical normalized greedy text from pinned llama-completion" + } + } + }, + { + "id": "family-paddleocr", + "family": "paddleocr", + "suites": [ + "llama-family-certification" + ], + "cadences": [ + "manual-full" + ], + "capability_tags": [ + "gguf", + "ocr" + ], + "artifact": { + "repo": "PaddlePaddle/PaddleOCR-VL-1.6-GGUF", + "revision": "511b09642bb324401f15f97cc23bc67e8f0a291d", + "selector": "BF16", + "files": [ + { + "path": "PaddleOCR-VL-1.6-GGUF.gguf", + "size_bytes": 935769056, + "sha256": "f3ae46ec885050acf4b3d31944431e1fd90d50664fb09126af4a3c050ba14ee8" + } + ] + }, + "certification": { + "class": "ocr", + "profile": "workload-oracle", + "cadences": [ + "manual-full" + ], + "execution": { + "trunk_layers": 18, + "mtp_layers": 0, + "activation_width": 1024, + "boundary_sweep_period": 0, + "speculative_policy": "disabled" + }, + "resources": { + "runner_role": "family-certify", + "cache_policy": "immutable-local", + "estimated_model_bytes": 933384192, + "startup_timeout_secs": 900 + }, + "notes": "Local full-model OCR and HTTP smoke with the pinned PaddleOCR projector and mandatory generated-image monolithic parity; projector and trunk remain colocated", + "evidence": { + "fixture": "scripts/generate-ocr-oracle-fixture.py#MESH 42", + "comparison": "normalized monolithic text parity plus independent MESH 42 label" + }, + "mmproj_artifact": { + "repo": "PaddlePaddle/PaddleOCR-VL-1.6-GGUF", + "revision": "511b09642bb324401f15f97cc23bc67e8f0a291d", + "selector": "BF16", + "files": [ + { + "path": "PaddleOCR-VL-1.6-GGUF-mmproj.gguf", + "size_bytes": 881770560, + "sha256": "204d757d7610d9b3faab10d506d69e5b244e32bf765e2bab2d0167e65e0a058a" + } + ] + } + } + }, + { + "id": "family-qwen3tts", + "family": "qwen3tts", + "suites": [ + "llama-family-certification" + ], + "cadences": [ + "manual-full" + ], + "capability_tags": [ + "gguf", + "speech_synthesis" + ], + "artifact": { + "repo": "ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF", + "revision": "ca27d74bc954b73dadab5b71ca265d87fc861a7c", + "selector": "Q8_0", + "files": [ + { + "path": "Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf", + "size_bytes": 1847874400, + "sha256": "ac7931aeb2e7aad1a6ed6602d353a5679c9d096b18ce8204ac730a8408d572e1" + } + ] + }, + "certification": { + "class": "speech_synthesis", + "profile": "workload-oracle", + "cadences": [ + "manual-full" + ], + "execution": { + "trunk_layers": 28, + "mtp_layers": 0, + "activation_width": 2048, + "boundary_sweep_period": 0, + "speculative_policy": "disabled" + }, + "resources": { + "runner_role": "family-certify", + "cache_policy": "immutable-local", + "estimated_model_bytes": 1841844224, + "startup_timeout_secs": 600 + }, + "notes": "Local full-model speech-synthesis and HTTP WAV smoke with deterministic pinned monolithic PCM parity; distributed staging is intentionally unsupported", + "evidence": { + "fixture": "scripts/skippy-tts-oracle.py#PROMPT", + "comparison": "fixed-seed PCM format and length; RMS error at most 2 percent and waveform cosine at least 0.9995" + }, + "mmproj_artifact": { + "repo": "ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF", + "revision": "ca27d74bc954b73dadab5b71ca265d87fc861a7c", + "selector": "Q8_0", + "files": [ + { + "path": "mmproj-Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf", + "size_bytes": 446422912, + "sha256": "6fd65188839bcd6ecc91b277ad471e22a0edfada4699a0fe82f1165c18cfcce2" + } + ] + } + } + }, + { + "id": "family-ultravox", + "family": "ultravox", + "suites": [ + "llama-family-certification" + ], + "cadences": [ + "manual-full" + ], + "capability_tags": [ + "gguf", + "speech_recognition" + ], + "artifact": { + "repo": "ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF", + "revision": "5390c7c41cbd6f261f7f205fc0c5ae61bbdca650", + "selector": "Q8_0", + "files": [ + { + "path": "Llama-3.2-1B-Instruct-Q8_0.gguf", + "size_bytes": 1321083008, + "sha256": "432f310a77f4650a88d0fd59ecdd7cebed8d684bafea53cbff0473542964f0c3" + } + ] + }, + "certification": { + "class": "speech_recognition", + "profile": "workload-oracle", + "cadences": [ + "manual-full" + ], + "execution": { + "trunk_layers": 16, + "mtp_layers": 0, + "activation_width": 2048, + "boundary_sweep_period": 0, + "speculative_policy": "disabled" + }, + "resources": { + "runner_role": "family-certify", + "cache_policy": "immutable-local", + "estimated_model_bytes": 1313251456, + "startup_timeout_secs": 600 + }, + "notes": "Local full-model audio-to-text and HTTP smoke with the pinned Ultravox projector and monolithic transcript parity; unlabeled WAV does not establish transcription accuracy", + "evidence": { + "fixture": "ci/llama-canary/fixtures/audio-smoke.wav", + "comparison": "identical normalized transcript against pinned llama-server; fixture is unlabeled" + }, + "mmproj_artifact": { + "repo": "ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF", + "revision": "5390c7c41cbd6f261f7f205fc0c5ae61bbdca650", + "selector": "F16", + "files": [ + { + "path": "mmproj-ultravox-v0_5-llama-3_2-1b-f16.gguf", + "size_bytes": 1371123616, + "sha256": "b34dde1835752949d6b960528269af93c92fec91c61ea0534fcc73f96c1ed8b2" + } + ] + } + } } ] } diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/events/query.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/query.rs index a584b9f0f0..401ef3d66e 100644 --- a/crates/mesh-llm-host-runtime/src/api/routes/logs/events/query.rs +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/query.rs @@ -291,17 +291,14 @@ fn parse_query(path: &str) -> Result { )); } } - "severity" => { - if audit_mode { - let severity = parse_audit_severity(&value)?; - if audit_selection.severity.is_some() { - return Err(LogsError::InvalidQuery("duplicate audit severity")); - } - audit_selection.severity = Some(severity); - } else { - return Err(LogsError::InvalidQuery("unknown event stream parameter")); + "severity" if audit_mode => { + let severity = parse_audit_severity(&value)?; + if audit_selection.severity.is_some() { + return Err(LogsError::InvalidQuery("duplicate audit severity")); } + audit_selection.severity = Some(severity); } + "severity" => return Err(LogsError::InvalidQuery("unknown event stream parameter")), "cursor" if cursor.is_none() && !audit_mode => { cursor = Some(Cursor::parse(nonempty(&value)?)?); } diff --git a/crates/mesh-llm-host-runtime/src/inference/skippy/mod.rs b/crates/mesh-llm-host-runtime/src/inference/skippy/mod.rs index 6d85384bb2..59c35ac0f3 100644 --- a/crates/mesh-llm-host-runtime/src/inference/skippy/mod.rs +++ b/crates/mesh-llm-host-runtime/src/inference/skippy/mod.rs @@ -28,11 +28,12 @@ use std::{ use anyhow::{Context, Result}; use async_trait::async_trait; use openai_frontend::{ + AudioResponse, AudioSpeechRequest, AudioTranscriptionRequest, AudioTranscriptionResponse, ChatCompletionRequest, ChatCompletionResponse, ChatCompletionStream, CompactingOpenAiBackend, - CompactionConfig, CompletionRequest, CompletionResponse, CompletionStream, - GuardedOpenAiBackend, GuardrailMode, GuardrailPolicy, GuardrailPolicyHandle, + CompactionConfig, CompletionRequest, CompletionResponse, CompletionStream, EmbeddingResponse, + EmbeddingsRequest, GuardedOpenAiBackend, GuardrailMode, GuardrailPolicy, GuardrailPolicyHandle, GuardrailTelemetrySink, ModelObject, OpenAiBackend, OpenAiHookPolicy, OpenAiRequestContext, - OpenAiResult, + OpenAiResult, RerankRequest, RerankResponse, }; use skippy_protocol::{FlashAttentionType, LoadMode, StageConfig, StageDevice, StageKvCacheConfig}; use skippy_runtime::{ModelInfo, MtpSource}; @@ -639,6 +640,26 @@ impl SkippyModelHandle { self.runtime.output_activation_boundary() } + pub(crate) fn workload_class(&self) -> Result { + if self.runtime.supports_speech_synthesis() { + return Ok(crate::mesh::ModelWorkloadClass::SpeechSynthesis); + } + let workload = self + .runtime + .workload_info() + .context("read loaded model workload contract")?; + Ok(match workload.kind { + skippy_runtime::ModelWorkload::CausalGeneration => { + crate::mesh::ModelWorkloadClass::CausalGeneration + } + skippy_runtime::ModelWorkload::Embedding => crate::mesh::ModelWorkloadClass::Embedding, + skippy_runtime::ModelWorkload::Rerank => crate::mesh::ModelWorkloadClass::Rerank, + skippy_runtime::ModelWorkload::EncoderDecoder => { + crate::mesh::ModelWorkloadClass::EncoderDecoder + } + }) + } + fn resolved_mtp_source( native_mtp_enabled: bool, native_mtp_draft_model_path: Option<&Path>, @@ -1186,6 +1207,46 @@ impl OpenAiBackend for SkippyModelHandle { ) -> OpenAiResult { self.backend.completion_stream(request, context).await } + + async fn embeddings( + &self, + request: EmbeddingsRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.embeddings(request, context).await + } + + async fn rerank( + &self, + request: RerankRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.rerank(request, context).await + } + + async fn audio_speech( + &self, + request: AudioSpeechRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.audio_speech(request, context).await + } + + async fn audio_transcription( + &self, + request: AudioTranscriptionRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.audio_transcription(request, context).await + } + + async fn audio_translation( + &self, + request: AudioTranscriptionRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.audio_translation(request, context).await + } } pub(crate) fn single_stage_config(options: &SkippyModelLoadOptions) -> Result { diff --git a/crates/mesh-llm-host-runtime/src/logging/openai_lifecycle.rs b/crates/mesh-llm-host-runtime/src/logging/openai_lifecycle.rs index 86aa96f5c8..07d3ac1ab9 100644 --- a/crates/mesh-llm-host-runtime/src/logging/openai_lifecycle.rs +++ b/crates/mesh-llm-host-runtime/src/logging/openai_lifecycle.rs @@ -777,6 +777,11 @@ fn terminal_outcome(result: OpenAiTerminalResult) -> TerminalOutcome { const fn operation_label(operation: OpenAiBackendOperation) -> &'static str { match operation { OpenAiBackendOperation::Models => "models", + OpenAiBackendOperation::Embeddings => "embeddings", + OpenAiBackendOperation::Rerank => "rerank", + OpenAiBackendOperation::AudioSpeech => "audio_speech", + OpenAiBackendOperation::AudioTranscription => "audio_transcription", + OpenAiBackendOperation::AudioTranslation => "audio_translation", OpenAiBackendOperation::ChatCompletion => "chat_completion", OpenAiBackendOperation::ChatCompletionStream => "chat_completion_stream", OpenAiBackendOperation::Completion => "completion", diff --git a/crates/mesh-llm-host-runtime/src/logging/request_metadata.rs b/crates/mesh-llm-host-runtime/src/logging/request_metadata.rs index 0cdb32542b..133167c80f 100644 --- a/crates/mesh-llm-host-runtime/src/logging/request_metadata.rs +++ b/crates/mesh-llm-host-runtime/src/logging/request_metadata.rs @@ -164,6 +164,11 @@ const fn openai_route_label(route: OpenAiFrontendRoute) -> Option<&'static str> OpenAiFrontendRoute::Healthz => Some("healthz"), OpenAiFrontendRoute::Readyz => Some("readyz"), OpenAiFrontendRoute::Models => Some("models"), + OpenAiFrontendRoute::Embeddings => Some("embeddings"), + OpenAiFrontendRoute::Rerank => Some("rerank"), + OpenAiFrontendRoute::AudioSpeech => Some("audio_speech"), + OpenAiFrontendRoute::AudioTranscriptions => Some("audio_transcriptions"), + OpenAiFrontendRoute::AudioTranslations => Some("audio_translations"), OpenAiFrontendRoute::ChatCompletions => Some("chat_completions"), OpenAiFrontendRoute::Completions => Some("completions"), OpenAiFrontendRoute::Responses => Some("responses"), diff --git a/crates/mesh-llm-host-runtime/src/mesh/identity_persistence.rs b/crates/mesh-llm-host-runtime/src/mesh/identity_persistence.rs index 5bb607462d..52e1ec83fa 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/identity_persistence.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/identity_persistence.rs @@ -249,7 +249,7 @@ pub(crate) async fn load_or_create_key() -> Result { pub fn default_node_key_path() -> Result { #[cfg(test)] if std::env::var_os("MESH_LLM_TEST_HOME").is_some() - && !std::env::var_os("MESH_LLM_NODE_KEY_PATH").is_some_and(|path| !path.is_empty()) + && std::env::var_os("MESH_LLM_NODE_KEY_PATH").is_none_or(|path| path.is_empty()) { return Ok(identity_home_dir().join(".mesh-llm").join("key")); } diff --git a/crates/mesh-llm-host-runtime/src/mesh/mod.rs b/crates/mesh-llm-host-runtime/src/mesh/mod.rs index c54a4f3fa9..4207e70b1c 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/mod.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/mod.rs @@ -6,9 +6,10 @@ //! latency-sensitive `skippy-stage/2` ALPN. pub use mesh_llm_types::mesh::{ - ModelDemand, ModelRuntimeDescriptor, ModelSourceKind, ServedModelDescriptor, - ServedModelIdentity, ServedModelMetadata, infer_available_model_descriptors, - infer_local_served_model_descriptor, infer_served_model_descriptors, + ModelDemand, ModelRuntimeDescriptor, ModelSourceKind, ModelWorkloadClass, + ServedModelDescriptor, ServedModelIdentity, ServedModelMetadata, + infer_available_model_descriptors, infer_local_served_model_descriptor, + infer_served_model_descriptors, }; use anyhow::{Context, Result}; diff --git a/crates/mesh-llm-host-runtime/src/models/profile.rs b/crates/mesh-llm-host-runtime/src/models/profile.rs index 6ed3c90783..167feca0be 100644 --- a/crates/mesh-llm-host-runtime/src/models/profile.rs +++ b/crates/mesh-llm-host-runtime/src/models/profile.rs @@ -34,6 +34,7 @@ pub(crate) fn served_model_metadata_for_path( let parameter_count_b = parameter_count.map(|total| total as f64 / 1e9); let kv_head_count = meta.effective_kv_head_count(); crate::mesh::ServedModelMetadata { + workload_class: None, architecture: non_empty(meta.architecture), parameter_size, parameter_count_b, diff --git a/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs b/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs index dc9e8266b6..ac141f5cf3 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs @@ -33,6 +33,13 @@ enum AutoRouteResolution { classification: Option, }, MediaUnsupported, + WorkloadUnsupported(mesh::ModelWorkloadClass), +} + +#[derive(Debug)] +enum AutoRouteRejection { + MediaUnsupported, + WorkloadUnsupported(mesh::ModelWorkloadClass), } struct IngressRouteContext<'a> { @@ -223,6 +230,7 @@ async fn resolve_auto_routed_model( required_tokens: Option, affinity: &affinity::AffinityRouter, ) -> AutoRouteResolution { + let requested_workload = request_workload_class(&request.client_path); // An explicitly named model routes to itself. The automatic directive (in // either spelling) resolves here, so `mesh` reaches the same // media-capability filter and readiness/affinity selection as `auto` — @@ -231,6 +239,11 @@ async fn resolve_auto_routed_model( if let Some(model) = request.model_name.as_deref() && !automatic::is_directive(model) { + if let Some(workload) = requested_workload + && !model_satisfies_request_workload(model, workload, &request.client_path, descriptors) + { + return AutoRouteResolution::WorkloadUnsupported(workload); + } return AutoRouteResolution::Continue { effective_model: request.model_name.clone(), classification: None, @@ -238,22 +251,28 @@ async fn resolve_auto_routed_model( } request.ensure_body_json(); - let Some(body_json) = request.body_json.as_ref() else { + let body_json = request.body_json.as_ref(); + if body_json.is_none() && requested_workload.is_none() { return AutoRouteResolution::Continue { effective_model: None, classification: None, }; - }; + } automatic::warn_if_deprecated_alias(request.model_name.as_deref()); - let mode = automatic::serving_mode(automatic::AutomaticRequest { - model: request.model_name.as_deref(), - // The forwarded path: `/v1/responses` has already been normalised onto - // chat completions by this point, so it stays committee-eligible. - path: &request.path, - body: body_json, - }); + let mode = body_json.map_or( + automatic::ServingMode::SingleModel(automatic::SingleModelReason::NonChatRequest), + |body| { + automatic::serving_mode(automatic::AutomaticRequest { + model: request.model_name.as_deref(), + // The forwarded path: `/v1/responses` has already been normalised onto + // chat completions by this point, so it stays committee-eligible. + path: &request.path, + body, + }) + }, + ); match mode { // Committee mode keeps the directive as the effective model so the MoA // gateway picks the request up. @@ -271,10 +290,33 @@ async fn resolve_auto_routed_model( ), } - let classification = router::classify(body_json); - let media = router::media_requirements(body_json); - let available_models = + let classification = body_json.map_or_else( + || router::Classification { + category: router::Category::Chat, + complexity: router::Complexity::Quick, + needs_tools: false, + has_media_inputs: is_audio_upload_path(&request.client_path), + }, + router::classify, + ); + let media = body_json.map_or_else( + || router::MediaRequirements { + has_media: is_audio_upload_path(&request.client_path), + needs_vision: false, + needs_audio: is_audio_upload_path(&request.client_path), + }, + router::media_requirements, + ); + let mut available_models = collect_available_models_for_auto_route(node, targets, plugin_manager).await; + if let Some(workload) = requested_workload { + available_models.retain(|model| { + model_satisfies_request_workload(model, workload, &request.client_path, descriptors) + }); + if available_models.is_empty() { + return AutoRouteResolution::WorkloadUnsupported(workload); + } + } let metrics = node.routing_metrics(); let available: Vec> = available_models .iter() @@ -318,6 +360,40 @@ async fn resolve_auto_routed_model( } } +fn is_audio_upload_path(path: &str) -> bool { + matches!( + path.split('?').next().unwrap_or(path), + "/v1/audio/transcriptions" | "/v1/audio/translations" + ) +} + +fn model_satisfies_request_workload( + model: &str, + workload: mesh::ModelWorkloadClass, + path: &str, + descriptors: &[mesh::ServedModelDescriptor], +) -> bool { + if is_audio_upload_path(path) { + proxy::model_satisfies_audio_upload_workload(model, descriptors) + } else { + proxy::model_satisfies_workload_class(model, workload, descriptors) + } +} + +fn request_workload_class(path: &str) -> Option { + match path.split('?').next().unwrap_or(path) { + "/v1/chat/completions" + | "/v1/completions" + | "/v1/responses" + | "/v1/audio/transcriptions" + | "/v1/audio/translations" => Some(mesh::ModelWorkloadClass::CausalGeneration), + "/v1/embeddings" => Some(mesh::ModelWorkloadClass::Embedding), + "/v1/rerank" => Some(mesh::ModelWorkloadClass::Rerank), + "/v1/audio/speech" => Some(mesh::ModelWorkloadClass::SpeechSynthesis), + _ => None, + } +} + async fn auto_route_pool_for_ready_models<'a>( node: &mesh::Node, targets: &election::ModelTargets, @@ -813,7 +889,7 @@ async fn prepare_auto_route_decision( request: &mut proxy::BufferedHttpRequest, ctx: &IngressRouteContext<'_>, descriptors: &[crate::mesh::ServedModelDescriptor], -) -> Result { +) -> Result { let required_tokens = proxy::request_context_budget(request); match resolve_auto_routed_model( ctx.node, @@ -840,10 +916,30 @@ async fn prepare_auto_route_decision( required_tokens, }) } - AutoRouteResolution::MediaUnsupported => Err(()), + AutoRouteResolution::MediaUnsupported => Err(AutoRouteRejection::MediaUnsupported), + AutoRouteResolution::WorkloadUnsupported(workload) => { + Err(AutoRouteRejection::WorkloadUnsupported(workload)) + } } } +async fn send_workload_unsupported( + tcp_stream: ClientStream, + workload: mesh::ModelWorkloadClass, + path: &str, + route_observer: OpenAiRouteObserver<'_>, +) -> proxy::RouteDispatchOutcome { + let message = if is_audio_upload_path(path) { + "no served model advertises support for this audio-to-text endpoint".to_string() + } else { + format!("no served model advertises the required {workload:?} workload") + }; + response_outcome( + 422, + proxy::send_error_observed(tcp_stream, 422, &message, route_observer).await, + ) +} + async fn send_media_unsupported( tcp_stream: ClientStream, route_observer: OpenAiRouteObserver<'_>, @@ -1089,11 +1185,22 @@ async fn handle_buffered_api_request( let decision = match prepare_auto_route_decision(&mut request, &ctx.route, &descriptors).await { Ok(decision) => decision, - Err(()) => { + Err(AutoRouteRejection::MediaUnsupported) => { let outcome = send_media_unsupported(tcp_stream, lifecycle.route_observer()).await; lifecycle.terminal(terminal_outcome_for_dispatch(outcome)); return; } + Err(AutoRouteRejection::WorkloadUnsupported(workload)) => { + let outcome = send_workload_unsupported( + tcp_stream, + workload, + &request.client_path, + lifecycle.route_observer(), + ) + .await; + lifecycle.terminal(terminal_outcome_for_dispatch(outcome)); + return; + } }; let mut routing_model = decision.effective_model.clone(); diff --git a/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/automatic_routing.rs b/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/automatic_routing.rs index 76be59f11e..4210bccd28 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/automatic_routing.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/automatic_routing.rs @@ -43,6 +43,32 @@ fn descriptor(model: &str, vision: bool, audio: bool) -> mesh::ServedModelDescri } } +fn workload_descriptor( + model: &str, + workload_class: mesh::ModelWorkloadClass, +) -> mesh::ServedModelDescriptor { + mesh::ServedModelDescriptor { + metadata: Some(mesh::ServedModelMetadata { + workload_class: Some(workload_class), + ..Default::default() + }), + ..descriptor(model, false, false) + } +} + +fn audio_workload_descriptor( + model: &str, + workload_class: mesh::ModelWorkloadClass, +) -> mesh::ServedModelDescriptor { + mesh::ServedModelDescriptor { + metadata: Some(mesh::ServedModelMetadata { + workload_class: Some(workload_class), + ..Default::default() + }), + ..descriptor(model, false, true) + } +} + fn request_with_body(model: Option<&str>, body: &serde_json::Value) -> proxy::BufferedHttpRequest { let body = serde_json::to_vec(body).expect("serialize body"); let raw = format!( @@ -145,6 +171,268 @@ async fn resolve( .await } +async fn resolve_path( + path: &str, + model: Option<&str>, + body: &serde_json::Value, + node: &mesh::Node, + targets: &election::ModelTargets, + descriptors: &[mesh::ServedModelDescriptor], +) -> AutoRouteResolution { + let mut request = request_with_body(model, body); + request.path = path.to_string(); + request.client_path = path.to_string(); + let affinity = affinity::AffinityRouter::new(); + resolve_auto_routed_model( + node, + &mut request, + targets, + None, + descriptors, + None, + &affinity, + ) + .await +} + +#[test] +fn endpoint_paths_map_to_their_required_workload_classes() { + assert_eq!( + super::super::ingress::request_workload_class("/v1/embeddings?trace=1"), + Some(mesh::ModelWorkloadClass::Embedding) + ); + assert_eq!( + super::super::ingress::request_workload_class("/v1/rerank"), + Some(mesh::ModelWorkloadClass::Rerank) + ); + assert_eq!( + super::super::ingress::request_workload_class("/v1/audio/speech"), + Some(mesh::ModelWorkloadClass::SpeechSynthesis) + ); + assert_eq!( + super::super::ingress::request_workload_class("/v1/audio/transcriptions"), + Some(mesh::ModelWorkloadClass::CausalGeneration) + ); + assert_eq!( + super::super::ingress::request_workload_class("/v1/audio/translations?trace=1"), + Some(mesh::ModelWorkloadClass::CausalGeneration) + ); +} + +#[tokio::test] +async fn audio_upload_routes_only_to_an_advertised_audio_workload() { + let (node, targets) = + node_serving(&["legacy-audio", "chat-only", "tts", "audio-to-text"]).await; + let descriptors = vec![ + descriptor("legacy-audio", false, true), + workload_descriptor("chat-only", mesh::ModelWorkloadClass::CausalGeneration), + audio_workload_descriptor("tts", mesh::ModelWorkloadClass::SpeechSynthesis), + audio_workload_descriptor("audio-to-text", mesh::ModelWorkloadClass::CausalGeneration), + ]; + for path in ["/v1/audio/transcriptions", "/v1/audio/translations?trace=1"] { + for requested_model in [automatic::DIRECTIVE, "audio-to-text"] { + let body = serde_json::json!({ "model": requested_model }); + let resolution = resolve_path( + path, + Some(requested_model), + &body, + &node, + &targets, + &descriptors, + ) + .await; + match resolution { + AutoRouteResolution::Continue { + effective_model, .. + } => assert_eq!(effective_model.as_deref(), Some("audio-to-text")), + AutoRouteResolution::WorkloadUnsupported(workload) => { + panic!("advertised audio-to-text model was rejected for {workload:?}") + } + AutoRouteResolution::MediaUnsupported => { + panic!("advertised audio-to-text model was rejected for media") + } + } + } + } +} + +#[tokio::test] +async fn audio_upload_rejects_legacy_audio_for_explicit_and_auto_routing() { + let (node, targets) = node_serving(&["legacy-audio"]).await; + let descriptors = vec![descriptor("legacy-audio", false, true)]; + + for path in ["/v1/audio/transcriptions", "/v1/audio/translations"] { + for requested_model in ["legacy-audio", automatic::DIRECTIVE] { + let body = serde_json::json!({ "model": requested_model }); + let resolution = resolve_path( + path, + Some(requested_model), + &body, + &node, + &targets, + &descriptors, + ) + .await; + assert!(matches!( + resolution, + AutoRouteResolution::WorkloadUnsupported( + mesh::ModelWorkloadClass::CausalGeneration + ) + )); + } + } +} + +#[tokio::test] +async fn audio_upload_requires_runtime_verified_audio_capability() { + let (node, targets) = node_serving(&["unverified-audio", "text-only"]).await; + let mut unverified = audio_workload_descriptor( + "unverified-audio", + mesh::ModelWorkloadClass::CausalGeneration, + ); + unverified.capabilities_known = false; + let descriptors = vec![ + unverified, + workload_descriptor("text-only", mesh::ModelWorkloadClass::CausalGeneration), + ]; + let body = serde_json::json!({ "model": automatic::DIRECTIVE }); + + let resolution = resolve_path( + "/v1/audio/transcriptions", + Some(automatic::DIRECTIVE), + &body, + &node, + &targets, + &descriptors, + ) + .await; + assert!(matches!( + resolution, + AutoRouteResolution::WorkloadUnsupported(mesh::ModelWorkloadClass::CausalGeneration) + )); +} + +#[tokio::test] +async fn legacy_chat_audio_remains_routable() { + let (node, targets) = node_serving(&["legacy-audio"]).await; + let descriptors = vec![descriptor("legacy-audio", false, true)]; + let body = serde_json::json!({ + "model": automatic::DIRECTIVE, + "messages": [{ + "role": "user", + "content": [{ + "type": "input_audio", + "input_audio": { "data": "AA==", "format": "wav" }, + }], + }], + }); + + let resolution = resolve_path( + "/v1/chat/completions", + Some(automatic::DIRECTIVE), + &body, + &node, + &targets, + &descriptors, + ) + .await; + match resolution { + AutoRouteResolution::Continue { + effective_model, .. + } => assert_eq!(effective_model.as_deref(), Some("legacy-audio")), + AutoRouteResolution::WorkloadUnsupported(workload) => { + panic!("legacy chat-audio was rejected for {workload:?}") + } + AutoRouteResolution::MediaUnsupported => panic!("legacy model advertises audio input"), + } +} + +#[tokio::test] +async fn embedding_auto_route_selects_only_an_embedding_model() { + let (node, targets) = node_serving(&["chat-model", "embed-model"]).await; + let descriptors = vec![ + workload_descriptor("chat-model", mesh::ModelWorkloadClass::CausalGeneration), + workload_descriptor("embed-model", mesh::ModelWorkloadClass::Embedding), + ]; + let body = serde_json::json!({ + "model": automatic::DIRECTIVE, + "input": ["alpha", "beta"], + }); + + let resolution = resolve_path( + "/v1/embeddings", + Some(automatic::DIRECTIVE), + &body, + &node, + &targets, + &descriptors, + ) + .await; + + match resolution { + AutoRouteResolution::Continue { + effective_model, .. + } => assert_eq!(effective_model.as_deref(), Some("embed-model")), + AutoRouteResolution::WorkloadUnsupported(workload) => { + panic!("embedding model was advertised but {workload:?} was rejected") + } + AutoRouteResolution::MediaUnsupported => panic!("embedding request has no media"), + } +} + +#[tokio::test] +async fn non_chat_auto_route_fails_closed_for_legacy_descriptors() { + let (node, targets) = node_serving(&["legacy-chat-model"]).await; + let descriptors = vec![descriptor("legacy-chat-model", false, false)]; + let body = serde_json::json!({ + "model": automatic::DIRECTIVE, + "input": "alpha", + }); + + let resolution = resolve_path( + "/v1/embeddings", + Some(automatic::DIRECTIVE), + &body, + &node, + &targets, + &descriptors, + ) + .await; + + assert!(matches!( + resolution, + AutoRouteResolution::WorkloadUnsupported(mesh::ModelWorkloadClass::Embedding) + )); +} + +#[tokio::test] +async fn explicitly_named_model_must_advertise_the_endpoint_workload() { + let (node, targets) = node_serving(&["chat-model"]).await; + let descriptors = vec![workload_descriptor( + "chat-model", + mesh::ModelWorkloadClass::CausalGeneration, + )]; + let body = serde_json::json!({ + "model": "chat-model", + "input": "alpha", + }); + + let resolution = resolve_path( + "/v1/embeddings", + Some("chat-model"), + &body, + &node, + &targets, + &descriptors, + ) + .await; + + assert!(matches!( + resolution, + AutoRouteResolution::WorkloadUnsupported(mesh::ModelWorkloadClass::Embedding) + )); +} + #[tokio::test] async fn plain_text_directive_stays_on_the_committee() { let (node, targets) = node_serving(&["vision-model", "text-model"]).await; @@ -167,6 +455,9 @@ async fn plain_text_directive_stays_on_the_committee() { AutoRouteResolution::Continue { effective_model, .. } => assert_eq!(effective_model.as_deref(), Some(automatic::DIRECTIVE)), + AutoRouteResolution::WorkloadUnsupported(workload) => { + panic!("generation request unexpectedly rejected for {workload:?}") + } AutoRouteResolution::MediaUnsupported => panic!("text request is not a media failure"), } } @@ -199,6 +490,9 @@ async fn image_request_resolves_to_a_vision_capable_model() { Some("vision-model"), "an image request must resolve to the vision-capable model, not the directive" ), + AutoRouteResolution::WorkloadUnsupported(workload) => { + panic!("generation request unexpectedly rejected for {workload:?}") + } AutoRouteResolution::MediaUnsupported => { panic!("a vision-capable model is served, so this must not fail") } @@ -252,6 +546,9 @@ async fn deprecated_alias_behaves_exactly_like_the_directive() { AutoRouteResolution::Continue { effective_model, .. } => assert_eq!(effective_model.as_deref(), Some(automatic::DIRECTIVE)), + AutoRouteResolution::WorkloadUnsupported(workload) => { + panic!("generation request unexpectedly rejected for {workload:?}") + } AutoRouteResolution::MediaUnsupported => panic!("text request is not a media failure"), } } @@ -292,6 +589,9 @@ async fn streaming_directive_resolves_to_a_single_model() { "must resolve to a served model, got {model}" ); } + AutoRouteResolution::WorkloadUnsupported(workload) => { + panic!("generation request unexpectedly rejected for {workload:?}") + } AutoRouteResolution::MediaUnsupported => panic!("no media in this request"), } } @@ -318,6 +618,9 @@ async fn model_less_request_resolves_to_a_single_model() { "a model-less request must not silently convene a committee" ); } + AutoRouteResolution::WorkloadUnsupported(workload) => { + panic!("generation request unexpectedly rejected for {workload:?}") + } AutoRouteResolution::MediaUnsupported => panic!("no media in this request"), } } @@ -443,6 +746,9 @@ async fn image_request_selects_a_vision_model_served_only_by_a_peer() { Some("remote-vision-model"), "the only vision-capable model is on the peer and must still be chosen" ), + AutoRouteResolution::WorkloadUnsupported(workload) => { + panic!("generation request unexpectedly rejected for {workload:?}") + } AutoRouteResolution::MediaUnsupported => { panic!("a peer serves a vision model, so this must not be refused") } @@ -472,6 +778,9 @@ async fn text_request_still_convenes_a_committee_across_two_nodes() { AutoRouteResolution::Continue { effective_model, .. } => assert_eq!(effective_model.as_deref(), Some(automatic::DIRECTIVE)), + AutoRouteResolution::WorkloadUnsupported(workload) => { + panic!("generation request unexpectedly rejected for {workload:?}") + } AutoRouteResolution::MediaUnsupported => panic!("text request is not a media failure"), } } @@ -526,6 +835,9 @@ async fn an_explicitly_named_model_is_never_reinterpreted() { AutoRouteResolution::Continue { effective_model, .. } => assert_eq!(effective_model.as_deref(), Some("text-model")), + AutoRouteResolution::WorkloadUnsupported(workload) => { + panic!("generation request unexpectedly rejected for {workload:?}") + } AutoRouteResolution::MediaUnsupported => panic!("explicit routing is untouched"), } } @@ -676,6 +988,9 @@ async fn a_non_chat_endpoint_resolves_to_a_single_model() { "a non-chat request must not convene a committee it cannot fan out" ); } + AutoRouteResolution::WorkloadUnsupported(workload) => { + panic!("generation request unexpectedly rejected for {workload:?}") + } AutoRouteResolution::MediaUnsupported => panic!("no media in this request"), } } diff --git a/crates/mesh-llm-host-runtime/src/network/openai/request_parse.rs b/crates/mesh-llm-host-runtime/src/network/openai/request_parse.rs index 996957a219..84070abc7a 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/request_parse.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/request_parse.rs @@ -10,6 +10,9 @@ use super::request_normalize::{ }; use super::routing_rank::descriptor_for_model; +mod audio_multipart; +use audio_multipart::{multipart_model_field, multipart_model_value_range}; + pub(crate) const MAX_HEADER_BYTES: usize = 64 * 1024; /// Private lifecycle ownership assertion used only on trusted mesh forwarding. /// @@ -20,8 +23,10 @@ pub(crate) const MAX_HEADER_BYTES: usize = 64 * 1024; pub(crate) const RAW_LIFECYCLE_OWNER_HEADER: &str = "x-mesh-llm-raw-lifecycle"; pub(super) const MAX_BODY_BYTES: usize = 8 * 1024 * 1024; const MAX_OBJECT_UPLOAD_BODY_BYTES: usize = 64 * 1024 * 1024; +const MAX_AUDIO_UPLOAD_BODY_BYTES: usize = 64 * 1024 * 1024 + 64 * 1024; const MAX_CHUNKED_WIRE_BYTES: usize = MAX_BODY_BYTES * 6 + 64 * 1024; const MAX_OBJECT_UPLOAD_CHUNKED_WIRE_BYTES: usize = MAX_OBJECT_UPLOAD_BODY_BYTES * 6 + 64 * 1024; +const MAX_AUDIO_UPLOAD_CHUNKED_WIRE_BYTES: usize = MAX_AUDIO_UPLOAD_BODY_BYTES * 6 + 64 * 1024; pub(super) const MAX_HEADERS: usize = 64; const CRLF: &[u8] = b"\r\n"; const LF: &[u8] = b"\n"; @@ -48,6 +53,7 @@ struct ParsedHeaders { path: String, request_id: RequestId, content_length: Option, + content_type: Option, is_chunked: bool, expects_continue: bool, correlation_id: Option, @@ -138,6 +144,12 @@ impl BufferedHttpRequest { is_tokenize_request(&self.method, &self.path) } + /// Multipart audio bytes are encoded media, not prompt text. The proxy + /// cannot infer their eventual model context size from the wire length. + pub fn is_audio_upload_request(&self) -> bool { + self.method == "POST" && is_audio_upload_path(&self.client_path) + } + pub fn ensure_body_json(&mut self) { if self.body_json.is_none() && !self.body_json_attempted { self.body_json = self @@ -354,6 +366,12 @@ where .map_err(|error| OpenAiRequestReadError::after_headers(error, &parsed))? .to_owned(), ) + } else if is_audio_upload_path(&parsed.path) { + match parsed.content_type.as_deref() { + Some(content_type) => multipart_model_field(content_type, &body) + .map_err(|error| OpenAiRequestReadError::after_headers(error, &parsed))?, + None => None, + } } else { metadata.as_ref().and_then(|value| value.model.clone()) }; @@ -536,11 +554,24 @@ fn body_limits_for_path(path: &str, default: HttpReadLimits) -> HttpReadLimits { max_body_bytes: MAX_OBJECT_UPLOAD_BODY_BYTES, max_chunked_wire_bytes: MAX_OBJECT_UPLOAD_CHUNKED_WIRE_BYTES, } + } else if is_audio_upload_path(path_only) { + HttpReadLimits { + max_header_bytes: default.max_header_bytes, + max_body_bytes: MAX_AUDIO_UPLOAD_BODY_BYTES, + max_chunked_wire_bytes: MAX_AUDIO_UPLOAD_CHUNKED_WIRE_BYTES, + } } else { default } } +fn is_audio_upload_path(path: &str) -> bool { + matches!( + path.split('?').next().unwrap_or(path), + "/v1/audio/transcriptions" | "/v1/audio/translations" + ) +} + fn finalize_forwarded_request( mut raw: Vec, header_end: usize, @@ -651,6 +682,7 @@ where let mut is_chunked = false; let mut expects_continue = false; let mut correlation_id = None; + let mut content_type = None; for header in req.headers.iter() { if header.name.eq_ignore_ascii_case("content-length") { @@ -671,6 +703,8 @@ where expects_continue = val .split(',') .any(|part| part.trim().eq_ignore_ascii_case("100-continue")); + } else if header.name.eq_ignore_ascii_case("content-type") { + content_type = std::str::from_utf8(header.value).ok().map(str::to_string); } else if header.name.eq_ignore_ascii_case("x-correlation-id") || header.name.eq_ignore_ascii_case("x-request-id") || header.name.eq_ignore_ascii_case("correlation-id") @@ -692,6 +726,7 @@ where path, request_id: request_id_from_headers(req.headers), content_length, + content_type, is_chunked, expects_continue, correlation_id, @@ -997,33 +1032,27 @@ pub fn inject_mesh_hooks_flag(raw: &mut Vec, enabled: bool) { *raw = result; } -/// Rewrite the JSON body `model` field and rebuild Content-Length. -pub fn rewrite_model_field(request: &mut BufferedHttpRequest, model: &str) { - let Some(header_end) = request - .raw - .windows(4) - .position(|w| w == b"\r\n\r\n") - .map(|i| i + 4) - else { - return; - }; - - let Ok(mut body) = serde_json::from_slice::(&request.raw[header_end..]) - else { - return; - }; - let Some(object) = body.as_object_mut() else { - return; - }; - - object.insert( - "model".to_string(), - serde_json::Value::String(model.to_string()), - ); - let Ok(new_body) = serde_json::to_vec(&body) else { - return; +fn content_type_from_request(raw: &[u8]) -> Option { + let mut headers = [httparse::EMPTY_HEADER; MAX_HEADERS]; + let mut parsed = httparse::Request::new(&mut headers); + let httparse::Status::Complete(_) = parsed.parse(raw).ok()? else { + return None; }; + parsed + .headers + .iter() + .find(|header| header.name.eq_ignore_ascii_case("content-type")) + .and_then(|header| std::str::from_utf8(header.value).ok()) + .map(str::to_string) +} +fn rebuild_request_body( + request: &mut BufferedHttpRequest, + header_end: usize, + new_body: Vec, + body_json: Option, + model: &str, +) { let headers = std::str::from_utf8(&request.raw[..header_end - 4]).unwrap_or(""); let mut rebuilt = String::new(); for line in headers.split("\r\n") { @@ -1038,15 +1067,54 @@ pub fn rewrite_model_field(request: &mut BufferedHttpRequest, model: &str) { let mut raw = rebuilt.into_bytes(); raw.extend_from_slice(&new_body); - request.raw = raw; request.body_len_bytes = new_body.len(); request.body_bytes = Some(new_body); - request.body_json = Some(body); + request.body_json = body_json; request.body_json_attempted = true; request.model_name = Some(model.to_string()); } +/// Rewrite the JSON or multipart `model` field and rebuild Content-Length. +pub fn rewrite_model_field(request: &mut BufferedHttpRequest, model: &str) { + let Some(header_end) = request + .raw + .windows(4) + .position(|w| w == b"\r\n\r\n") + .map(|i| i + 4) + else { + return; + }; + + if let Ok(mut body) = serde_json::from_slice::(&request.raw[header_end..]) { + let Some(object) = body.as_object_mut() else { + return; + }; + object.insert( + "model".to_string(), + serde_json::Value::String(model.to_string()), + ); + let Ok(new_body) = serde_json::to_vec(&body) else { + return; + }; + rebuild_request_body(request, header_end, new_body, Some(body), model); + return; + } + + let Some(content_type) = content_type_from_request(&request.raw) else { + return; + }; + let original = &request.raw[header_end..]; + let Ok(Some(range)) = multipart_model_value_range(&content_type, original) else { + return; + }; + let mut new_body = Vec::with_capacity(original.len() - range.len() + model.len()); + new_body.extend_from_slice(&original[..range.start]); + new_body.extend_from_slice(model.as_bytes()); + new_body.extend_from_slice(&original[range.end..]); + rebuild_request_body(request, header_end, new_body, None, model); +} + pub fn is_models_list_request(method: &str, path: &str) -> bool { let path = path.split('?').next().unwrap_or(path); method == "GET" && (path == "/v1/models" || path == "/models") diff --git a/crates/mesh-llm-host-runtime/src/network/openai/request_parse/audio_multipart.rs b/crates/mesh-llm-host-runtime/src/network/openai/request_parse/audio_multipart.rs new file mode 100644 index 0000000000..ddd26af4ab --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/network/openai/request_parse/audio_multipart.rs @@ -0,0 +1,180 @@ +use anyhow::{Context, Result, bail}; + +use super::{CRLF, CRLF_HEADER_TERMINATOR, MAX_HEADER_BYTES}; + +pub(super) fn multipart_boundary(content_type: &str) -> Option<&str> { + let mut parts = content_type.split(';'); + if !parts + .next()? + .trim() + .eq_ignore_ascii_case("multipart/form-data") + { + return None; + } + let boundary = parts.find_map(|part| { + let (name, value) = part.trim().split_once('=')?; + name.trim() + .eq_ignore_ascii_case("boundary") + .then_some(value.trim().trim_matches('"')) + })?; + let valid = !boundary.is_empty() + && boundary.len() <= 70 + && boundary + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"'()+_,-./:=?".contains(&byte)); + valid.then_some(boundary) +} + +fn find_subslice(haystack: &[u8], needle: &[u8]) -> Option { + (!needle.is_empty()) + .then(|| { + haystack + .windows(needle.len()) + .position(|window| window == needle) + }) + .flatten() +} + +fn find_multipart_boundary(body: &[u8], marker: &[u8], from: usize) -> Option { + let mut cursor = from; + while let Some(offset) = find_subslice(&body[cursor..], marker) { + let start = cursor + offset; + let suffix = body.get(start + marker.len()..start + marker.len() + 2); + if suffix == Some(CRLF) || suffix == Some(b"--") { + return Some(start); + } + cursor = start + 1; + } + None +} + +fn disposition_parameters(value: &str) -> Result> { + let mut parameters = Vec::new(); + let mut start = 0; + let mut quoted = false; + let mut escaped = false; + for (index, byte) in value.bytes().enumerate() { + if escaped { + escaped = false; + continue; + } + match byte { + b'\\' if quoted => escaped = true, + b'"' => quoted = !quoted, + b';' if !quoted => { + parameters.push(&value[start..index]); + start = index + 1; + } + _ => {} + } + } + if quoted || escaped { + bail!("malformed multipart Content-Disposition"); + } + parameters.push(&value[start..]); + Ok(parameters) +} + +pub(super) fn multipart_part_is_model(headers: &str) -> Result { + let mut disposition = None; + for line in headers.split("\r\n") { + let Some((name, value)) = line.split_once(':') else { + continue; + }; + if name.eq_ignore_ascii_case("content-disposition") + && disposition.replace(value.trim()).is_some() + { + bail!("duplicate multipart Content-Disposition header"); + } + } + let Some(disposition) = disposition else { + return Ok(false); + }; + let parameters = disposition_parameters(disposition)?; + if !parameters[0].trim().eq_ignore_ascii_case("form-data") { + return Ok(false); + } + let mut field_name = None; + for parameter in parameters.iter().skip(1) { + let Some((name, value)) = parameter.trim().split_once('=') else { + continue; + }; + if name.trim().eq_ignore_ascii_case("name") { + if field_name.is_some() { + bail!("duplicate multipart field name parameter"); + } + let value = value.trim(); + let parsed = if let Some(inner) = value + .strip_prefix('"') + .and_then(|quoted| quoted.strip_suffix('"')) + { + inner + } else if !value.contains('"') { + value + } else { + bail!("malformed multipart field name parameter"); + }; + field_name = Some(parsed); + } + } + Ok(field_name == Some("model")) +} + +pub(super) fn multipart_model_value_range( + content_type: &str, + body: &[u8], +) -> Result>> { + let Some(boundary) = multipart_boundary(content_type) else { + return Ok(None); + }; + let delimiter = format!("--{boundary}").into_bytes(); + if !body.starts_with(&delimiter) { + bail!("multipart body must start with its declared boundary"); + } + let next_marker = [CRLF, &delimiter].concat(); + let mut cursor = 0; + let mut model_range = None; + loop { + let part_start = cursor + delimiter.len(); + if body.get(part_start..part_start + 2) == Some(b"--") { + return Ok(model_range); + } + if body.get(part_start..part_start + 2) != Some(CRLF) { + bail!("malformed multipart boundary"); + } + let content_start = part_start + CRLF.len(); + let header_search_end = content_start + .saturating_add(MAX_HEADER_BYTES + CRLF_HEADER_TERMINATOR.len()) + .min(body.len()); + let headers_end = find_subslice( + &body[content_start..header_search_end], + CRLF_HEADER_TERMINATOR, + ) + .map(|offset| content_start + offset) + .context("multipart part is missing a header terminator")?; + if headers_end.saturating_sub(content_start) > MAX_HEADER_BYTES { + bail!("multipart part headers exceed the limit"); + } + let headers = std::str::from_utf8(&body[content_start..headers_end]) + .context("multipart part headers are not UTF-8")?; + let is_model = multipart_part_is_model(headers)?; + let value_start = headers_end + CRLF_HEADER_TERMINATOR.len(); + let value_end = find_multipart_boundary(body, &next_marker, value_start) + .context("multipart part is missing its closing boundary")?; + if is_model { + if model_range.is_some() { + bail!("duplicate multipart model field"); + } + model_range = Some(value_start..value_end); + } + cursor = value_end + CRLF.len(); + } +} + +pub(super) fn multipart_model_field(content_type: &str, body: &[u8]) -> Result> { + let Some(range) = multipart_model_value_range(content_type, body)? else { + return Ok(None); + }; + let value = std::str::from_utf8(&body[range])?.trim(); + Ok((!value.is_empty() && value.len() <= 256).then(|| value.to_string())) +} diff --git a/crates/mesh-llm-host-runtime/src/network/openai/request_parse_tests.rs b/crates/mesh-llm-host-runtime/src/network/openai/request_parse_tests.rs index 85d12e7408..d04120c993 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/request_parse_tests.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/request_parse_tests.rs @@ -1,3 +1,4 @@ +use super::audio_multipart::{multipart_boundary, multipart_part_is_model}; use super::*; use tokio::io::AsyncWriteExt; use tokio::net::TcpListener; @@ -652,6 +653,42 @@ async fn test_read_http_request_allows_large_object_upload_body() { assert!(request.request_object_request_ids.is_empty()); } +#[tokio::test] +async fn test_read_http_request_allows_large_audio_upload_body() { + let file_bytes = vec![b'x'; MAX_BODY_BYTES + 1]; + let mut body = + b"--audio\r\nContent-Disposition: form-data; name=\"file\"; filename=\"large.wav\"\r\n\r\n" + .to_vec(); + body.extend_from_slice(&file_bytes); + body.extend_from_slice(b"\r\n--audio\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\naudio-model\r\n--audio--\r\n"); + let headers = format!( + "POST /v1/audio/transcriptions HTTP/1.1\r\nHost: localhost\r\nContent-Type: multipart/form-data; boundary=audio\r\nContent-Length: {}\r\n\r\n", + body.len() + ) + .into_bytes(); + + let request = read_request_from_parts(vec![headers, body.clone()]).await; + + assert_eq!(request.path, "/v1/audio/transcriptions"); + assert!(request.raw.ends_with(&body)); + assert_eq!(request.body_len_bytes, body.len()); + assert_eq!(request.model_name.as_deref(), Some("audio-model")); +} + +#[test] +fn audio_upload_limits_are_path_scoped() { + let audio = body_limits_for_path("/v1/audio/translations?trace=1", HTTP_READ_LIMITS); + assert_eq!(audio.max_body_bytes, MAX_AUDIO_UPLOAD_BODY_BYTES); + assert_eq!( + audio.max_chunked_wire_bytes, + MAX_AUDIO_UPLOAD_CHUNKED_WIRE_BYTES + ); + + let embedding = body_limits_for_path("/v1/embeddings", HTTP_READ_LIMITS); + assert_eq!(embedding.max_body_bytes, MAX_BODY_BYTES); + assert_eq!(embedding.max_chunked_wire_bytes, MAX_CHUNKED_WIRE_BYTES); +} + #[tokio::test] async fn test_read_http_request_expect_100_continue() { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -789,6 +826,152 @@ fn test_rewrite_model_field_updates_body_and_content_length() { assert_eq!(declared, request.body_len_bytes); } +#[tokio::test] +async fn multipart_model_is_parsed_and_rewritten_without_touching_file_bytes() { + const BOUNDARY: &str = "mesh-audio-boundary"; + // A boundary prefix inside binary content is not a multipart delimiter. + let file_bytes = [ + 0_u8, 255, 13, 10, b'-', b'-', b'm', b'e', b's', b'h', b'-', b'a', b'u', b'd', b'i', b'o', + b'-', b'b', b'o', b'u', b'n', b'd', b'a', b'r', b'y', b'X', 13, 10, 1, 2, 3, 128, + ]; + let mut body = format!( + "--{BOUNDARY}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"voice; name=model\"\r\nContent-Type: audio/wav\r\n\r\n" + ) + .into_bytes(); + body.extend_from_slice(&file_bytes); + body.extend_from_slice( + format!( + "\r\n--{BOUNDARY}\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\nauto\r\n--{BOUNDARY}--\r\n" + ) + .as_bytes(), + ); + let headers = format!( + "POST /v1/audio/transcriptions HTTP/1.1\r\nHost: localhost\r\nContent-Type: multipart/form-data; boundary=\"{BOUNDARY}\"\r\nContent-Length: {}\r\n\r\n", + body.len() + ) + .into_bytes(); + + let mut request = read_request_from_parts(vec![headers, body]).await; + assert_eq!(request.model_name.as_deref(), Some("auto")); + + rewrite_model_field(&mut request, "whisper-local"); + + assert_eq!(request.model_name.as_deref(), Some("whisper-local")); + assert!( + request + .raw + .windows(file_bytes.len()) + .any(|window| window == file_bytes) + ); + let header_end = request + .raw + .windows(4) + .position(|window| window == b"\r\n\r\n") + .unwrap() + + 4; + let content_type = content_type_from_request(&request.raw).unwrap(); + assert_eq!( + multipart_model_field(&content_type, &request.raw[header_end..]) + .unwrap() + .as_deref(), + Some("whisper-local") + ); + let declared = std::str::from_utf8(&request.raw[..header_end]) + .unwrap() + .lines() + .find_map(|line| { + line.split_once(':') + .filter(|(name, _)| name.eq_ignore_ascii_case("content-length")) + .and_then(|(_, value)| value.trim().parse::().ok()) + }) + .unwrap(); + assert_eq!(declared, request.raw.len() - header_end); + assert_eq!(declared, request.body_len_bytes); +} + +#[test] +fn multipart_parser_rejects_invalid_boundaries_and_oversized_model_values() { + assert!(multipart_boundary("multipart/form-data; boundary=bad space").is_none()); + assert!(multipart_boundary("application/json; boundary=mesh").is_none()); + + let boundary = "mesh"; + let body = format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\n{}\r\n--{boundary}--\r\n", + "x".repeat(257) + ); + assert!( + multipart_model_field( + &format!("multipart/form-data; boundary={boundary}"), + body.as_bytes() + ) + .unwrap() + .is_none() + ); +} + +#[tokio::test] +async fn duplicate_multipart_model_is_rejected_before_audio_routing() { + let boundary = "mesh-audio-boundary"; + let body = format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\nauto\r\n\ + --{boundary}\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\nother-model\r\n\ + --{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"voice.wav\"\r\n\r\nWAVE\r\n\ + --{boundary}--\r\n" + ); + let content_type = format!("multipart/form-data; boundary={boundary}"); + assert!( + multipart_model_field(&content_type, body.as_bytes()) + .unwrap_err() + .to_string() + .contains("duplicate multipart model field") + ); + + for path in ["/v1/audio/transcriptions", "/v1/audio/translations"] { + let request = format!( + "POST {path} HTTP/1.1\r\nHost: localhost\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ); + let (mut client, mut server) = tokio::io::duplex(request.len() + 1); + client.write_all(request.as_bytes()).await.unwrap(); + client.shutdown().await.unwrap(); + let error = read_http_request_with_plugin_manager_with_context(&mut server, None) + .await + .unwrap_err(); + assert_eq!(error.context().unwrap().client_path, path); + assert!( + error + .to_string() + .contains("duplicate multipart model field") + ); + } +} + +#[test] +fn multipart_model_scanner_rejects_non_initial_boundary() { + let body = b"binary--mesh\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\nauto\r\n--mesh--\r\n"; + let error = multipart_model_field("multipart/form-data; boundary=mesh", body).unwrap_err(); + assert!( + error + .to_string() + .contains("must start with its declared boundary") + ); +} + +#[test] +fn multipart_disposition_ignores_name_like_text_inside_quoted_filename() { + assert!( + !multipart_part_is_model( + "Content-Disposition: form-data; name=\"file\"; filename=\"voice; name=model\"" + ) + .unwrap() + ); + assert!(multipart_part_is_model("Content-Disposition: form-data; name=\"model\"").unwrap()); + assert!( + multipart_part_is_model("Content-Disposition: form-data; name=\"file\"; name=\"model\"") + .is_err() + ); +} + #[test] fn artifact_media_kind_is_closed_to_parsed_openai_json_routes() { let request = |path: &str, body: Option<&[u8]>| BufferedHttpRequest { diff --git a/crates/mesh-llm-host-runtime/src/network/openai/response/models.rs b/crates/mesh-llm-host-runtime/src/network/openai/response/models.rs index bce5e1e674..4df1bbed76 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/response/models.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/response/models.rs @@ -140,6 +140,9 @@ fn model_metadata_json( ) -> Option { let mut metadata = serde_json::Map::new(); let descriptor_metadata = descriptor.and_then(|descriptor| descriptor.metadata.as_ref()); + if let Some(value) = descriptor_metadata.and_then(|metadata| metadata.workload_class) { + metadata.insert("workload_class".to_string(), serde_json::json!(value)); + } if let Some(value) = descriptor_metadata.and_then(|metadata| metadata.architecture.as_ref()) { metadata.insert("architecture".to_string(), serde_json::json!(value)); } @@ -411,6 +414,7 @@ mod tests { let models = vec!["Qwen3-32B-Q4_K_M".to_string()]; let mut descriptor = local_gguf_descriptor(&models[0]); descriptor.metadata = Some(mesh::ServedModelMetadata { + workload_class: Some(mesh::ModelWorkloadClass::CausalGeneration), architecture: Some("qwen3".to_string()), parameter_size: Some("32B".to_string()), parameter_count_b: Some(32.0), @@ -433,6 +437,7 @@ mod tests { let body = models_list_json(&models, &[descriptor], &runtimes); let metadata = &body["data"][0]["metadata"]; + assert_eq!(metadata["workload_class"], "causal_generation"); assert_eq!(metadata["architecture"], "qwen3"); assert_eq!(metadata["parameter_size"], "32B"); diff --git a/crates/mesh-llm-host-runtime/src/network/openai/routing_rank.rs b/crates/mesh-llm-host-runtime/src/network/openai/routing_rank.rs index 952cf0603d..cdb5c69435 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/routing_rank.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/routing_rank.rs @@ -365,6 +365,59 @@ pub(crate) fn descriptor_metadata_for_model<'a>( descriptor_for_model(descriptors, model).and_then(|descriptor| descriptor.metadata.as_ref()) } +pub(crate) fn workload_class_for_model( + model: &str, + descriptors: &[mesh::ServedModelDescriptor], +) -> Option { + descriptor_metadata_for_model(model, descriptors).and_then(|metadata| metadata.workload_class) +} + +/// Checks the additive workload advertisement without breaking legacy chat +/// routing. An absent field is compatible only with generative endpoints: +/// older nodes predate workload classes and must never be assumed to support a +/// newly introduced non-chat response contract. +pub(crate) fn model_satisfies_workload_class( + model: &str, + requested: mesh::ModelWorkloadClass, + descriptors: &[mesh::ServedModelDescriptor], +) -> bool { + match (requested, workload_class_for_model(model, descriptors)) { + (mesh::ModelWorkloadClass::CausalGeneration, None) => true, + ( + mesh::ModelWorkloadClass::CausalGeneration, + Some( + mesh::ModelWorkloadClass::CausalGeneration + | mesh::ModelWorkloadClass::EncoderDecoder, + ), + ) => true, + (requested, Some(advertised)) => requested == advertised, + (_, None) => false, + } +} + +/// Audio uploads use a newer HTTP contract than chat requests with audio +/// parts. A legacy peer may advertise audio input but lack these endpoints, so +/// neither inferred capabilities nor an absent workload class can opt it in. +pub(crate) fn model_satisfies_audio_upload_workload( + model: &str, + descriptors: &[mesh::ServedModelDescriptor], +) -> bool { + descriptor_for_model(descriptors, model).is_some_and(|descriptor| { + descriptor.capabilities_known + && descriptor.capabilities.supports_audio_runtime() + && matches!( + descriptor + .metadata + .as_ref() + .and_then(|metadata| metadata.workload_class), + Some( + mesh::ModelWorkloadClass::CausalGeneration + | mesh::ModelWorkloadClass::EncoderDecoder + ) + ) + }) +} + #[cfg(test)] mod tests { use super::*; @@ -391,6 +444,83 @@ mod tests { ..local_gguf_descriptor(model_name) } } + + fn descriptor_with_workload( + model_name: &str, + workload_class: mesh::ModelWorkloadClass, + ) -> mesh::ServedModelDescriptor { + mesh::ServedModelDescriptor { + metadata: Some(mesh::ServedModelMetadata { + workload_class: Some(workload_class), + ..Default::default() + }), + ..local_gguf_descriptor(model_name) + } + } + + #[test] + fn legacy_descriptors_are_compatible_only_with_generation_routes() { + let descriptors = vec![local_gguf_descriptor("legacy")]; + + assert!(model_satisfies_workload_class( + "legacy", + mesh::ModelWorkloadClass::CausalGeneration, + &descriptors + )); + assert!(!model_satisfies_workload_class( + "legacy", + mesh::ModelWorkloadClass::Embedding, + &descriptors + )); + assert!(!model_satisfies_workload_class( + "legacy", + mesh::ModelWorkloadClass::SpeechSynthesis, + &descriptors + )); + } + + #[test] + fn workload_routes_require_an_exact_advertised_class() { + let descriptors = vec![ + descriptor_with_workload("embed", mesh::ModelWorkloadClass::Embedding), + descriptor_with_workload("rank", mesh::ModelWorkloadClass::Rerank), + ]; + + assert!(model_satisfies_workload_class( + "embed", + mesh::ModelWorkloadClass::Embedding, + &descriptors + )); + assert!(!model_satisfies_workload_class( + "embed", + mesh::ModelWorkloadClass::Rerank, + &descriptors + )); + assert!(model_satisfies_workload_class( + "rank", + mesh::ModelWorkloadClass::Rerank, + &descriptors + )); + } + + #[test] + fn encoder_decoder_models_can_serve_generation_routes() { + let descriptors = vec![descriptor_with_workload( + "t5", + mesh::ModelWorkloadClass::EncoderDecoder, + )]; + + assert!(model_satisfies_workload_class( + "t5", + mesh::ModelWorkloadClass::CausalGeneration, + &descriptors + )); + assert!(!model_satisfies_workload_class( + "t5", + mesh::ModelWorkloadClass::Embedding, + &descriptors + )); + } #[test] fn test_cached_auto_model_rejects_text_model_for_image_request() { let body = serde_json::json!({ diff --git a/crates/mesh-llm-host-runtime/src/network/openai/transport.rs b/crates/mesh-llm-host-runtime/src/network/openai/transport.rs index 896b6e9e2f..4bd90c830f 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/transport.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/transport.rs @@ -30,7 +30,8 @@ pub(crate) use super::response::{ send_json_with_status_and_headers_observed, send_models_list_with_descriptors, }; pub(crate) use super::routing_rank::{ - capabilities_for_model, descriptor_metadata_for_model, request_budget_tokens_from_parts, + capabilities_for_model, descriptor_metadata_for_model, model_satisfies_audio_upload_workload, + model_satisfies_workload_class, request_budget_tokens_from_parts, }; use super::response::{ @@ -172,11 +173,12 @@ pub(crate) async fn reject_legacy_lifecycle_request( ) } -/// Generation context is a property of decode requests, not capability RPCs. -/// A tokenizer request may carry a megabyte of source text while using no -/// target KV context at all. +/// Generation context cannot be estimated from every request body's byte size. +/// Tokenizer requests use no target KV context, and multipart audio bodies +/// contain encoded media rather than text tokens. The audio backend performs +/// the authoritative media/context validation after routing. pub(crate) fn request_context_budget(request: &BufferedHttpRequest) -> Option { - if request.is_tokenize_request() { + if request.is_tokenize_request() || request.is_audio_upload_request() { None } else { request_budget_tokens_from_parts(request.body_len_bytes, request.completion_tokens) diff --git a/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/routing.rs b/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/routing.rs index 7b6a8082b6..a1b57faf54 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/routing.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/routing.rs @@ -338,6 +338,55 @@ async fn remote_tokenizer_plan_routes_identity_model_without_context_rejection() Ok(()) } +#[tokio::test] +async fn remote_audio_upload_ignores_encoded_bytes_as_context_tokens() -> Result<()> { + let model = "acme/audio-model:Q4_K_M"; + let peer_id = iroh::EndpointId::from(iroh::SecretKey::generate().public()); + let node = test_node_with_remote_models(&[(model, peer_id)]).await; + let mut peer = test_peer_serving_model(peer_id, model); + peer.served_model_runtime = vec![mesh::ModelRuntimeDescriptor { + model_name: model.to_owned(), + identity_hash: None, + context_length: Some(8_192), + ready: true, + }]; + node.insert_test_peer(peer).await; + + for path in [ + "/v1/audio/transcriptions", + "/v1/audio/translations?verbose=1", + ] { + let mut request = large_tokenize_request(model); + request.path = path.to_owned(); + request.client_path = path.to_owned(); + request.body_len_bytes = 1_048_576; + let mistaken_text_budget = + request_budget_tokens_from_parts(request.body_len_bytes, request.completion_tokens); + assert!(mistaken_text_budget.is_some_and(|tokens| tokens > 8_192)); + assert_eq!(request_context_budget(&request), None); + + let ranked = order_remote_hosts_by_context( + &node, + model, + request_context_budget(&request), + std::slice::from_ref(&peer_id), + ) + .await; + assert_eq!( + ranked, + vec![peer_id], + "encoded audio must not reject {path}" + ); + } + + let mut text_request = large_tokenize_request(model); + text_request.path = "/v1/chat/completions".to_owned(); + text_request.client_path = text_request.path.clone(); + text_request.body_len_bytes = 1_048_576; + assert!(request_context_budget(&text_request).is_some()); + Ok(()) +} + #[test] fn tokenizer_effective_model_cannot_override_authoritative_identity() { let model = "acme/code-model:Q4_K_M"; diff --git a/crates/mesh-llm-host-runtime/src/protocol/convert.rs b/crates/mesh-llm-host-runtime/src/protocol/convert.rs index 35d86f09df..b1a27b4eed 100644 --- a/crates/mesh-llm-host-runtime/src/protocol/convert.rs +++ b/crates/mesh-llm-host-runtime/src/protocol/convert.rs @@ -311,6 +311,7 @@ fn local_model_metadata_to_proto( metadata: &crate::mesh::ServedModelMetadata, ) -> crate::proto::node::ServedModelMetadata { crate::proto::node::ServedModelMetadata { + workload_class: metadata.workload_class.map(local_workload_class_to_proto), architecture: metadata.architecture.clone(), parameter_size: metadata.parameter_size.clone(), parameter_count_b: metadata.parameter_count_b, @@ -330,6 +331,9 @@ fn proto_model_metadata_to_local( metadata: &crate::proto::node::ServedModelMetadata, ) -> crate::mesh::ServedModelMetadata { crate::mesh::ServedModelMetadata { + workload_class: metadata + .workload_class + .and_then(proto_workload_class_to_local), architecture: metadata.architecture.clone(), parameter_size: metadata.parameter_size.clone(), parameter_count_b: metadata.parameter_count_b, @@ -345,6 +349,33 @@ fn proto_model_metadata_to_local( } } +fn local_workload_class_to_proto(workload: crate::mesh::ModelWorkloadClass) -> i32 { + use crate::mesh::ModelWorkloadClass as Local; + use crate::proto::node::ModelWorkloadClass as Proto; + + match workload { + Local::CausalGeneration => Proto::CausalGeneration as i32, + Local::Embedding => Proto::Embedding as i32, + Local::Rerank => Proto::Rerank as i32, + Local::EncoderDecoder => Proto::EncoderDecoder as i32, + Local::SpeechSynthesis => Proto::SpeechSynthesis as i32, + } +} + +fn proto_workload_class_to_local(value: i32) -> Option { + use crate::mesh::ModelWorkloadClass as Local; + use crate::proto::node::ModelWorkloadClass as Proto; + + match Proto::try_from(value).ok()? { + Proto::Unspecified => None, + Proto::CausalGeneration => Some(Local::CausalGeneration), + Proto::Embedding => Some(Local::Embedding), + Proto::Rerank => Some(Local::Rerank), + Proto::EncoderDecoder => Some(Local::EncoderDecoder), + Proto::SpeechSynthesis => Some(Local::SpeechSynthesis), + } +} + fn runtime_descriptor_to_proto( descriptor: &crate::mesh::ModelRuntimeDescriptor, ) -> crate::proto::node::ModelRuntimeDescriptor { @@ -1352,6 +1383,45 @@ mod tests { use super::*; use crate::mesh::requirements::peer_release_attestation_status; + #[test] + fn workload_class_round_trips_through_additive_proto_metadata() { + for workload in [ + crate::mesh::ModelWorkloadClass::CausalGeneration, + crate::mesh::ModelWorkloadClass::Embedding, + crate::mesh::ModelWorkloadClass::Rerank, + crate::mesh::ModelWorkloadClass::EncoderDecoder, + crate::mesh::ModelWorkloadClass::SpeechSynthesis, + ] { + let local = crate::mesh::ServedModelMetadata { + workload_class: Some(workload), + architecture: Some("test".to_string()), + ..Default::default() + }; + + let proto = local_model_metadata_to_proto(&local); + let restored = proto_model_metadata_to_local(&proto); + + assert_eq!(restored.workload_class, Some(workload)); + assert_eq!(restored.architecture.as_deref(), Some("test")); + } + } + + #[test] + fn absent_or_unknown_proto_workload_class_is_legacy_compatible() { + let absent = crate::proto::node::ServedModelMetadata::default(); + assert_eq!(proto_model_metadata_to_local(&absent).workload_class, None); + + let unknown = crate::proto::node::ServedModelMetadata { + workload_class: Some(9_999), + ..Default::default() + }; + assert_eq!( + proto_model_metadata_to_local(&unknown).workload_class, + None, + "newer workload enum values must be ignored by older conversion code" + ); + } + #[test] fn proto_ann_to_local_preserves_malformed_release_attestation_for_later_rejection() { let proto = crate::proto::node::PeerAnnouncement { diff --git a/crates/mesh-llm-host-runtime/src/runtime/local.rs b/crates/mesh-llm-host-runtime/src/runtime/local.rs index 9ac15e46b2..34e5c732cf 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/local.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/local.rs @@ -96,6 +96,7 @@ pub(super) struct LocalRuntimeModelHandle { pub(super) context_length: u32, pub(super) slots: usize, pub(super) capabilities: models::ModelCapabilities, + pub(super) workload_class: mesh::ModelWorkloadClass, pub(super) inner: LocalRuntimeBackendHandle, } @@ -495,6 +496,7 @@ pub(super) async fn set_runtime_verified_served_model_capabilities( primary_model_name: &str, model_name: &str, capabilities: models::ModelCapabilities, + workload_class: mesh::ModelWorkloadClass, ) { let existing = node .served_model_descriptors() @@ -506,6 +508,7 @@ pub(super) async fn set_runtime_verified_served_model_capabilities( primary_model_name, model_name, capabilities, + workload_class, ); node.upsert_served_model_descriptor(descriptor).await; } @@ -515,6 +518,7 @@ pub(super) fn runtime_verified_served_model_descriptor( primary_model_name: &str, model_name: &str, capabilities: models::ModelCapabilities, + workload_class: mesh::ModelWorkloadClass, ) -> mesh::ServedModelDescriptor { let mut descriptor = existing.unwrap_or_else(|| mesh::ServedModelDescriptor { identity: mesh::ServedModelIdentity { @@ -534,6 +538,10 @@ pub(super) fn runtime_verified_served_model_descriptor( descriptor.capabilities_known = true; descriptor.capabilities = capabilities; descriptor + .metadata + .get_or_insert_with(Default::default) + .workload_class = Some(workload_class); + descriptor } pub(super) async fn remove_serving_assignment(node: &mesh::Node, model_name: &str) { @@ -790,6 +798,7 @@ async fn start_local_skippy_model( }) .await .context("join load skippy direct GGUF task")??; + let workload_class = skippy_model.workload_class()?; let _ = emit_event(OutputEvent::ModelLoaded { model: model_name.clone(), bytes: None, @@ -805,6 +814,7 @@ async fn start_local_skippy_model( context_length, slots: plan.slots, capabilities, + workload_class, inner: LocalRuntimeBackendHandle::Skippy { model: skippy_model, http, @@ -952,6 +962,7 @@ async fn start_local_package_v2_model( }) .await .context("join load skippy package-v2 task")??; + let workload_class = handle.workload_class()?; let _ = emit_event(OutputEvent::ModelLoaded { model: model_ref, bytes: None, @@ -967,6 +978,7 @@ async fn start_local_package_v2_model( context_length, slots: plan.slots, capabilities, + workload_class, inner: LocalRuntimeBackendHandle::Skippy { model: handle, http, diff --git a/crates/mesh-llm-host-runtime/src/runtime/local_split/loading.rs b/crates/mesh-llm-host-runtime/src/runtime/local_split/loading.rs index db19d4473e..334efd78fd 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/local_split/loading.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/local_split/loading.rs @@ -344,6 +344,7 @@ pub(super) async fn load_split_runtime_generation_inner( context_length: spec.ctx_size, slots: spec.slots, capabilities, + workload_class: mesh::ModelWorkloadClass::CausalGeneration, inner: LocalRuntimeBackendHandle::Skippy { model: handle, http, diff --git a/crates/mesh-llm-host-runtime/src/runtime/local_split/test_support.rs b/crates/mesh-llm-host-runtime/src/runtime/local_split/test_support.rs index 2f39ce8371..2b6d449bf5 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/local_split/test_support.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/local_split/test_support.rs @@ -836,6 +836,7 @@ fn runtime_verified_served_model_descriptor_preserves_identity_and_updates_capab "Qwen3VL-2B-Instruct-Q4_K_M", "Qwen3VL-2B-Instruct-Q4_K_M", capabilities, + mesh::ModelWorkloadClass::CausalGeneration, ); assert_eq!( @@ -858,6 +859,7 @@ fn runtime_verified_served_model_descriptor_builds_fallback_identity() { "Primary", "Runtime", models::ModelCapabilities::default(), + mesh::ModelWorkloadClass::Embedding, ); assert_eq!(descriptor.identity.model_name, "Runtime"); diff --git a/crates/mesh-llm-host-runtime/src/runtime/model_lifecycle.rs b/crates/mesh-llm-host-runtime/src/runtime/model_lifecycle.rs index d57a7fd4fd..5f1a39bcdb 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/model_lifecycle.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/model_lifecycle.rs @@ -5,8 +5,8 @@ use super::{ ModelTargetReconciliationCapacityState, ModelTargetReconciliationInput, ModelTargetReconciliationPolicy, ModelTargetReconciliationState, RunAutoRuntimeLoopContext, RunAutoRuntimeState, RuntimeCapacityReservation, RuntimeEvent, RuntimeInstanceRegistry, - RuntimeOperationalEvent, RuntimeOptions, RuntimeUnloadCandidate, RuntimeUnloadOwner, - ShutdownRuntimeLoadedModelsContext, StartupModelSpec, StartupReadyReporter, + RuntimeModelRegistration, RuntimeOperationalEvent, RuntimeOptions, RuntimeUnloadCandidate, + RuntimeUnloadOwner, ShutdownRuntimeLoadedModelsContext, StartupModelSpec, StartupReadyReporter, add_runtime_local_target, add_serving_assignment, find_remote_catalog_model_exact_blocking, local_process_payload, next_runtime_instance_id, plan_model_target_reconciliation, publish_runtime_llama_slots, publish_runtime_llama_unavailable, diff --git a/crates/mesh-llm-host-runtime/src/runtime/model_lifecycle/load.rs b/crates/mesh-llm-host-runtime/src/runtime/model_lifecycle/load.rs index 7b2de92ff5..30e516522b 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/model_lifecycle/load.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/model_lifecycle/load.rs @@ -329,11 +329,14 @@ async fn finish_runtime_model_load( register_runtime_instance( ctx.runtime_instance_registry, ctx.node, - ctx.primary_model_name, - &loaded_name, - &instance_id, - Some(handle.context_length), - handle.capabilities, + RuntimeModelRegistration { + primary_model_name: ctx.primary_model_name, + model_name: &loaded_name, + instance_id: &instance_id, + context_length: Some(handle.context_length), + capabilities: handle.capabilities, + workload_class: handle.workload_class, + }, ) .await; ctx.node diff --git a/crates/mesh-llm-host-runtime/src/runtime/runtime_registry.rs b/crates/mesh-llm-host-runtime/src/runtime/runtime_registry.rs index 0e7aaeaca9..50c65c851b 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/runtime_registry.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/runtime_registry.rs @@ -15,6 +15,15 @@ use std::sync::Arc; pub(super) type RuntimeInstanceRegistry = Arc>>>>; +pub(super) struct RuntimeModelRegistration<'a> { + pub primary_model_name: &'a str, + pub model_name: &'a str, + pub instance_id: &'a str, + pub context_length: Option, + pub capabilities: models::ModelCapabilities, + pub workload_class: mesh::ModelWorkloadClass, +} + pub(super) fn next_runtime_instance_id(next_sequence: &mut u64) -> String { let instance_id = format!("runtime-{}", *next_sequence); *next_sequence = next_sequence.saturating_add(1); @@ -67,12 +76,16 @@ pub(super) fn reserve_runtime_capacity_for_model( pub(super) async fn register_runtime_instance( registry: &RuntimeInstanceRegistry, node: &mesh::Node, - primary_model_name: &str, - model_name: &str, - instance_id: &str, - context_length: Option, - capabilities: models::ModelCapabilities, + registration: RuntimeModelRegistration<'_>, ) { + let RuntimeModelRegistration { + primary_model_name, + model_name, + instance_id, + context_length, + capabilities, + workload_class, + } = registration; let (was_empty, context_changed, next_context) = { let mut guard = registry.lock().await; let instances = guard.entry(model_name.to_string()).or_default(); @@ -93,6 +106,7 @@ pub(super) async fn register_runtime_instance( primary_model_name, model_name, capabilities, + workload_class, ) .await; advertise_model_ready(node, primary_model_name, model_name, "").await; diff --git a/crates/mesh-llm-host-runtime/src/runtime/startup_handles.rs b/crates/mesh-llm-host-runtime/src/runtime/startup_handles.rs index 4e1d646316..1f72d14ece 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/startup_handles.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/startup_handles.rs @@ -6,10 +6,10 @@ use super::{ DASHBOARD_CONTEXT_USAGE_REFRESH_INTERVAL, DashboardContextUsage, InitialPromptMode, InstanceLifecycleRecord, InstanceLifecycleState, LocalRuntimeModelHandle, LocalRuntimeModelStartSpec, OpenAiGuardrailPolicyHandle, RuntimeCapacityLedger, - RuntimeCapacityReservation, RuntimeInstanceRegistry, RuntimeOperationalEvent, - RuntimeResourcePlanningProfile, SPLIT_STANDBY_RETRY_INTERVAL, SplitCoordinatorAck, - SplitCoordinatorEvent, SplitRuntimeReason, SplitRuntimeStart, StartupPinnedGpuTarget, - StartupRuntimePlan, add_runtime_local_target, local_process_payload, + RuntimeCapacityReservation, RuntimeInstanceRegistry, RuntimeModelRegistration, + RuntimeOperationalEvent, RuntimeResourcePlanningProfile, SPLIT_STANDBY_RETRY_INTERVAL, + SplitCoordinatorAck, SplitCoordinatorEvent, SplitRuntimeReason, SplitRuntimeStart, + StartupPinnedGpuTarget, StartupRuntimePlan, add_runtime_local_target, local_process_payload, publish_runtime_llama_slots, publish_runtime_llama_unavailable, record_runtime_operational_event, record_runtime_operational_event_with_context, refresh_dashboard_context_usage, register_runtime_instance, remove_dashboard_context_usage, @@ -487,11 +487,14 @@ pub(super) async fn startup_register_loaded_runtime( register_runtime_instance( ctx.runtime_instance_registry, ctx.node, - ctx.primary_model_name, - loaded_name, - ctx.instance_id, - Some(handle.context_length), - handle.capabilities, + RuntimeModelRegistration { + primary_model_name: ctx.primary_model_name, + model_name: loaded_name, + instance_id: ctx.instance_id, + context_length: Some(handle.context_length), + capabilities: handle.capabilities, + workload_class: handle.workload_class, + }, ) .await; let payload = local_process_payload( diff --git a/crates/mesh-llm-host-runtime/src/runtime/tests/model_lifecycle.rs b/crates/mesh-llm-host-runtime/src/runtime/tests/model_lifecycle.rs index d5d98fafd3..f467ddcc69 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/tests/model_lifecycle.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/tests/model_lifecycle.rs @@ -386,21 +386,27 @@ async fn register_runtime_instance_preserves_existing_known_descriptor_capabilit register_runtime_instance( ®istry, &node, - vision_model, - vision_model, - "runtime-vision", - Some(8192), - vision_capabilities, + RuntimeModelRegistration { + primary_model_name: vision_model, + model_name: vision_model, + instance_id: "runtime-vision", + context_length: Some(8192), + capabilities: vision_capabilities, + workload_class: mesh::ModelWorkloadClass::CausalGeneration, + }, ) .await; register_runtime_instance( ®istry, &node, - vision_model, - text_model, - "runtime-text", - Some(8192), - models::ModelCapabilities::default(), + RuntimeModelRegistration { + primary_model_name: vision_model, + model_name: text_model, + instance_id: "runtime-text", + context_length: Some(8192), + capabilities: models::ModelCapabilities::default(), + workload_class: mesh::ModelWorkloadClass::Embedding, + }, ) .await; diff --git a/crates/mesh-llm-protocol/proto/node.proto b/crates/mesh-llm-protocol/proto/node.proto index 7a2465f2a2..56b539d3b6 100644 --- a/crates/mesh-llm-protocol/proto/node.proto +++ b/crates/mesh-llm-protocol/proto/node.proto @@ -261,6 +261,16 @@ message ServedModelMetadata { optional uint32 kv_head_count = 10; optional uint32 expert_count = 11; optional uint32 active_expert_count = 12; + optional ModelWorkloadClass workload_class = 13; +} + +enum ModelWorkloadClass { + MODEL_WORKLOAD_CLASS_UNSPECIFIED = 0; + MODEL_WORKLOAD_CLASS_CAUSAL_GENERATION = 1; + MODEL_WORKLOAD_CLASS_EMBEDDING = 2; + MODEL_WORKLOAD_CLASS_RERANK = 3; + MODEL_WORKLOAD_CLASS_ENCODER_DECODER = 4; + MODEL_WORKLOAD_CLASS_SPEECH_SYNTHESIS = 5; } message ServedModelIdentity { diff --git a/crates/mesh-llm-protocol/src/proto/node.rs b/crates/mesh-llm-protocol/src/proto/node.rs index d812ee6420..3c40fb593a 100644 --- a/crates/mesh-llm-protocol/src/proto/node.rs +++ b/crates/mesh-llm-protocol/src/proto/node.rs @@ -321,6 +321,8 @@ pub struct ServedModelMetadata { pub expert_count: ::core::option::Option, #[prost(uint32, optional, tag = "12")] pub active_expert_count: ::core::option::Option, + #[prost(enumeration = "ModelWorkloadClass", optional, tag = "13")] + pub workload_class: ::core::option::Option, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ServedModelIdentity { @@ -983,6 +985,44 @@ impl ModelSourceKind { } } } +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ModelWorkloadClass { + Unspecified = 0, + CausalGeneration = 1, + Embedding = 2, + Rerank = 3, + EncoderDecoder = 4, + SpeechSynthesis = 5, +} +impl ModelWorkloadClass { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Unspecified => "MODEL_WORKLOAD_CLASS_UNSPECIFIED", + Self::CausalGeneration => "MODEL_WORKLOAD_CLASS_CAUSAL_GENERATION", + Self::Embedding => "MODEL_WORKLOAD_CLASS_EMBEDDING", + Self::Rerank => "MODEL_WORKLOAD_CLASS_RERANK", + Self::EncoderDecoder => "MODEL_WORKLOAD_CLASS_ENCODER_DECODER", + Self::SpeechSynthesis => "MODEL_WORKLOAD_CLASS_SPEECH_SYNTHESIS", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "MODEL_WORKLOAD_CLASS_UNSPECIFIED" => Some(Self::Unspecified), + "MODEL_WORKLOAD_CLASS_CAUSAL_GENERATION" => Some(Self::CausalGeneration), + "MODEL_WORKLOAD_CLASS_EMBEDDING" => Some(Self::Embedding), + "MODEL_WORKLOAD_CLASS_RERANK" => Some(Self::Rerank), + "MODEL_WORKLOAD_CLASS_ENCODER_DECODER" => Some(Self::EncoderDecoder), + "MODEL_WORKLOAD_CLASS_SPEECH_SYNTHESIS" => Some(Self::SpeechSynthesis), + _ => None, + } + } +} /// Shared enum #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] diff --git a/crates/mesh-llm-types/src/mesh/mod.rs b/crates/mesh-llm-types/src/mesh/mod.rs index 74fcd1955e..869fa3c403 100644 --- a/crates/mesh-llm-types/src/mesh/mod.rs +++ b/crates/mesh-llm-types/src/mesh/mod.rs @@ -50,8 +50,28 @@ fn is_false(value: &bool) -> bool { !*value } +/// The primary inference contract exposed by a loaded model runtime. +/// +/// This is deliberately separate from multimodal capabilities: a causal model +/// can accept image or audio inputs, while an embedding or reranking model has +/// a different output contract even when its input is text-only. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ModelWorkloadClass { + CausalGeneration, + Embedding, + Rerank, + EncoderDecoder, + SpeechSynthesis, +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] pub struct ServedModelMetadata { + /// `None` is the mixed-version representation for nodes that predate + /// workload advertisement. Callers must not treat it as evidence that a + /// non-chat endpoint is supported. + #[serde(skip_serializing_if = "Option::is_none")] + pub workload_class: Option, #[serde(skip_serializing_if = "Option::is_none")] pub architecture: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -80,7 +100,8 @@ pub struct ServedModelMetadata { impl ServedModelMetadata { pub fn is_empty(&self) -> bool { - self.architecture.is_none() + self.workload_class.is_none() + && self.architecture.is_none() && self.parameter_size.is_none() && self.parameter_count_b.is_none() && self.quant.is_none() diff --git a/crates/openai-frontend/Cargo.toml b/crates/openai-frontend/Cargo.toml index 70ca397a78..619bc0a0d5 100644 --- a/crates/openai-frontend/Cargo.toml +++ b/crates/openai-frontend/Cargo.toml @@ -9,7 +9,8 @@ homepage = "https://github.com/Mesh-LLM/mesh-llm" [dependencies] async-trait = "0.1" -axum = "0.8" +axum = { version = "0.8", features = ["multipart"] } +base64 = "0.22" futures-core = "0.3" futures-util = "0.3" mesh-llm-guardrails = { path = "../mesh-llm-guardrails", version = "0.76.1" } diff --git a/crates/openai-frontend/README.md b/crates/openai-frontend/README.md index 733d36a316..a0d5eb27dd 100644 --- a/crates/openai-frontend/README.md +++ b/crates/openai-frontend/README.md @@ -6,8 +6,8 @@ entry points. This crate owns the public API shapes and route machinery that should not be duplicated inside `skippy-server`. Stage server code should provide a thin backend adapter that implements the frontend trait, while this crate handles -request/response JSON, OpenAI-style errors, `/v1/models`, chat completions, and -streaming Server-Sent Events framing. Mesh uses this as the single OpenAI +request/response JSON, OpenAI-style errors, model discovery, generation, +embedding, rerank, audio, and streaming Server-Sent Events framing. Mesh uses this as the single OpenAI surface for embedded single-stage and stage-split serving. Mesh-local compatibility wrappers should stay thin. Request normalization, @@ -31,6 +31,11 @@ For the concrete benchy command and contract, see | `POST /v1/chat/completions` | Supported | Handles streaming and non-streaming response shapes. | | `POST /v1/completions` | Supported | Handles streaming and non-streaming response shapes. | | `POST /v1/responses` | Supported | Adapts OpenAI responses requests onto chat/completion backend calls and preserves response metadata where possible. | +| `POST /v1/embeddings` | Supported | String and token inputs, float or base64 vectors, usage accounting. Runtime support is model-gated. | +| `POST /v1/rerank` | Supported | Cross-encoder query/document scoring with optional document return. Runtime support is model-gated. | +| `POST /v1/audio/speech` | Supported | Binary response with format-specific content type. The native backend currently produces WAV or PCM and accepts only `voice: "default"`; speaker selection fails with a structured unsupported error. | +| `POST /v1/audio/transcriptions` | Supported | Bounded multipart audio upload with JSON or text response. Runtime support is model-gated. | +| `POST /v1/audio/translations` | Supported | Bounded multipart audio upload translated to English, with JSON or text response. Runtime support is model-gated. | | `GET /health` / `GET /healthz` | Supported | Lightweight liveness probes for hosts and CI smoke tests. | | `GET /readyz` | Supported | Backend readiness probe that verifies model discovery through `OpenAiBackend::models`. | | Server-Sent Events | Supported | Emits OpenAI-style JSON chunks and `[DONE]`. | @@ -52,13 +57,14 @@ For the concrete benchy command and contract, see | Client nonce | Supported | Accepts `x-capsule-client-nonce` only when it is exactly one valid UUIDv4; a missing, invalid, non-UUIDv4, or duplicated value is replaced with a freshly minted UUIDv4. When this frontend mints the value it stamps `x-capsule-nonce-origin: frontend`; a forwarded (client-supplied) nonce carries no origin marker, and any inbound `x-capsule-nonce-origin` is always stripped so a caller cannot forge it. Both headers are echoed on covered responses: the axum router and, via the host runtime's forwarding rebuild, the public `:9337` proxy JSON/SSE paths (including the pipeline/MoA strong-model path and remapped upstream error responses). Locally synthesized error responses (e.g. no-target `503`s and `/v1/models` listings) do not yet carry the headers; threading the request nonce onto those senders is a cross-cutting signature change tracked as a follow-up. The origin marker asserts only that *this* frontend minted the value, not that it is the original ingress for a remote-routed request. | | Backend timeout | Supported | Configurable via `OpenAiFrontendConfig` or the `MESH_OPENAI_BACKEND_TIMEOUT_SECS` environment variable; defaults to 600 seconds (`0` disables it) and maps timeouts to OpenAI-shaped 504 errors. | | Agent session header | Supported | Set `MESH_AGENT_SESSION_HEADER` to accept a trusted upstream header as the stable agent-session identity. | -| embeddings/rerank/infill/audio/vision | Out of scope | Not needed for staged text benchmark entrypoints. | +| Vision input | Supported | Preserved through chat/Responses content parts and executed by projector-backed runtimes. | +| Non-chat staging | Fail closed | Embedding, rerank, encoder-decoder, and speech-synthesis models currently require an unsplit full-model runtime. | ## Shape ```mermaid flowchart TB - C["OpenAI-compatible client
chat, completions, responses"] --> R["openai-frontend
Axum routes"] + C["OpenAI-compatible client
generation, embeddings, audio"] --> R["openai-frontend
Axum routes"] R --> Parse["request parsing
validation
normalization
OpenAI errors"] Parse --> B["OpenAiBackend implementation"] B --> Local["embedded single-stage
skippy runtime"] @@ -116,6 +122,9 @@ chain remains backend-owned. | `/v1/chat/completions` | Parse common and advanced fields, stream/non-stream envelopes | Tokenization, sampling, stop handling, usage, feature execution | Supported with backend feature guards | | `/v1/completions` | Prompt parsing and response envelopes | Token prompts, sampling, stop handling, usage | Supported with backend feature guards | | `/v1/responses` | Translate request/response shapes onto the backend contract | Execute the resulting chat/completion request | Supported | +| `/v1/embeddings` | Parse OpenAI input/encoding shapes and serialize vectors | Tokenize or accept token IDs, pool, normalize, and report usage | Supported for compatible local full models | +| `/v1/rerank` | Parse query/documents and serialize ranked results | Execute classifier scoring and report usage | Supported for compatible local full models | +| `/v1/audio/*` | Parse JSON or bounded multipart bodies; return JSON, text, or binary media | Execute TTS or projector-backed speech recognition | Supported for compatible local full models | | HTTP operations | Health/readiness, fallbacks, payload limits, content-type handling | Model readiness and backend timeouts | Supported | | Streaming | SSE chunks, `[DONE]`, cancellation context | Produce deltas, usage, and optional logprob/tool metadata | Supported with backend feature guards | | Chat templates | Preserve OpenAI message shape | Apply model-aware chat templates through the skippy ABI | Backend-owned | @@ -130,7 +139,7 @@ chain remains backend-owned. | Logprobs | Parse and preserve request/response shape | Expose logits/probabilities | Frontend ready; backend-gated | | Tools/function calling | Parse and preserve tool schemas and tool-call response shape | Generate tool calls | Frontend ready; backend-gated | | JSON schema/grammar | Parse and preserve `response_format` | Constrained decoding | Frontend ready; backend-gated | -| Embeddings/rerank/infill/audio | Route fallback/error handling | Runtime implementation if reintroduced | Out of current scope | +| Non-chat workload gates | Preserve request/response contracts and structured errors | Probe the native model class and reject unsupported stage shapes | Supported | | Metrics | Request IDs and tracing context | Stage/OpenAI telemetry emitted to `metrics-server` | Supported | ## Stage-Server Integration @@ -144,3 +153,6 @@ small adapter: That keeps `serve-openai` and the embedded mesh path thin: parse or build the runtime config, construct the backend, pass it to `openai_frontend::router`, and serve the Axum app. + +See [`docs/NON_CHAT_MODELS.md`](../../docs/NON_CHAT_MODELS.md) for endpoint +examples, execution boundaries, and certification policy. diff --git a/crates/openai-frontend/src/audio.rs b/crates/openai-frontend/src/audio.rs new file mode 100644 index 0000000000..9a54607b57 --- /dev/null +++ b/crates/openai-frontend/src/audio.rs @@ -0,0 +1,130 @@ +use serde::{Deserialize, Serialize}; + +use crate::{OpenAiError, OpenAiResult}; + +const MAX_AUDIO_BYTES: usize = 64 * 1024 * 1024; + +#[derive(Debug, Clone, Copy, Default, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum AudioFormat { + #[default] + Mp3, + Opus, + Aac, + Flac, + Wav, + Pcm, +} + +impl AudioFormat { + pub const fn content_type(self) -> &'static str { + match self { + Self::Mp3 => "audio/mpeg", + Self::Opus => "audio/opus", + Self::Aac => "audio/aac", + Self::Flac => "audio/flac", + Self::Wav => "audio/wav", + Self::Pcm => "audio/pcm", + } + } +} + +#[derive(Debug, Clone, Deserialize, PartialEq)] +pub struct AudioSpeechRequest { + pub model: String, + pub input: String, + pub voice: String, + #[serde(default)] + pub response_format: AudioFormat, + #[serde(default = "default_speed")] + pub speed: f32, +} + +fn default_speed() -> f32 { + 1.0 +} + +impl AudioSpeechRequest { + pub fn validate(&self) -> OpenAiResult<()> { + if self.model.trim().is_empty() || self.input.is_empty() || self.voice.trim().is_empty() { + return Err(OpenAiError::invalid_request( + "model, input, and voice must not be empty", + )); + } + if !(0.25..=4.0).contains(&self.speed) || !self.speed.is_finite() { + return Err(OpenAiError::invalid_request( + "speed must be a finite value between 0.25 and 4.0", + )); + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AudioResponse { + pub bytes: Vec, + pub content_type: String, +} + +impl AudioResponse { + pub fn new(bytes: Vec, content_type: impl Into) -> OpenAiResult { + if bytes.is_empty() { + return Err(OpenAiError::backend( + "audio backend returned an empty payload", + )); + } + Ok(Self { + bytes, + content_type: content_type.into(), + }) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct AudioTranscriptionRequest { + pub model: String, + pub file: Vec, + pub filename: Option, + pub language: Option, + pub prompt: Option, + pub response_format: String, + pub temperature: Option, +} + +impl AudioTranscriptionRequest { + pub const MAX_FILE_BYTES: usize = MAX_AUDIO_BYTES; + + pub fn validate(&self) -> OpenAiResult<()> { + if self.model.trim().is_empty() { + return Err(OpenAiError::invalid_request("model must not be empty")); + } + if self.file.is_empty() { + return Err(OpenAiError::invalid_request("audio file must not be empty")); + } + if self.file.len() > Self::MAX_FILE_BYTES { + return Err(OpenAiError::payload_too_large(format!( + "audio file exceeds the {} byte limit", + Self::MAX_FILE_BYTES + ))); + } + if !matches!(self.response_format.as_str(), "json" | "text") { + return Err(OpenAiError::unsupported( + "response_format must be 'json' or 'text'", + )); + } + if self + .temperature + .is_some_and(|value| !value.is_finite() || value < 0.0) + { + return Err(OpenAiError::invalid_request( + "temperature must be a finite non-negative value", + )); + } + Ok(()) + } +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct AudioTranscriptionResponse { + pub text: String, +} diff --git a/crates/openai-frontend/src/backend.rs b/crates/openai-frontend/src/backend.rs index e6aae05d03..61d41234ad 100644 --- a/crates/openai-frontend/src/backend.rs +++ b/crates/openai-frontend/src/backend.rs @@ -11,11 +11,16 @@ use futures_core::Stream; use tokio::sync::Notify; use crate::{ + audio::{ + AudioResponse, AudioSpeechRequest, AudioTranscriptionRequest, AudioTranscriptionResponse, + }, chat::{ChatCompletionChunk, ChatCompletionRequest, ChatCompletionResponse}, completions::{CompletionChunk, CompletionRequest, CompletionResponse}, + embeddings::{EmbeddingResponse, EmbeddingsRequest}, errors::OpenAiError, lifecycle::RequestId, models::ModelObject, + rerank::{RerankRequest, RerankResponse}, }; pub type ChatCompletionStream = @@ -184,6 +189,56 @@ pub trait OpenAiBackend: Send + Sync + 'static { "/v1/completions streaming is not supported by this backend", )) } + + async fn embeddings( + &self, + _request: EmbeddingsRequest, + _context: OpenAiRequestContext, + ) -> OpenAiResult { + Err(OpenAiError::unsupported( + "/v1/embeddings is not supported by this backend", + )) + } + + async fn rerank( + &self, + _request: RerankRequest, + _context: OpenAiRequestContext, + ) -> OpenAiResult { + Err(OpenAiError::unsupported( + "/v1/rerank is not supported by this backend", + )) + } + + async fn audio_speech( + &self, + _request: AudioSpeechRequest, + _context: OpenAiRequestContext, + ) -> OpenAiResult { + Err(OpenAiError::unsupported( + "/v1/audio/speech is not supported by this backend", + )) + } + + async fn audio_transcription( + &self, + _request: AudioTranscriptionRequest, + _context: OpenAiRequestContext, + ) -> OpenAiResult { + Err(OpenAiError::unsupported( + "/v1/audio/transcriptions is not supported by this backend", + )) + } + + async fn audio_translation( + &self, + _request: AudioTranscriptionRequest, + _context: OpenAiRequestContext, + ) -> OpenAiResult { + Err(OpenAiError::unsupported( + "/v1/audio/translations is not supported by this backend", + )) + } } pub(crate) type SharedBackend = Arc; diff --git a/crates/openai-frontend/src/chat.rs b/crates/openai-frontend/src/chat.rs index a6ac07f00f..f1babd0732 100644 --- a/crates/openai-frontend/src/chat.rs +++ b/crates/openai-frontend/src/chat.rs @@ -141,10 +141,10 @@ fn validate_tools_value(value: &Value) -> Result<(), String> { let Some(function) = tool.get("function").and_then(Value::as_object) else { return Err(format!("tools[{index}].function must be an object")); }; - if !function + if function .get("name") .and_then(Value::as_str) - .is_some_and(|name| !name.trim().is_empty()) + .is_none_or(|name| name.trim().is_empty()) { return Err(format!( "tools[{index}].function.name must be a non-empty string" diff --git a/crates/openai-frontend/src/embeddings.rs b/crates/openai-frontend/src/embeddings.rs new file mode 100644 index 0000000000..b8c4f33096 --- /dev/null +++ b/crates/openai-frontend/src/embeddings.rs @@ -0,0 +1,198 @@ +use base64::{Engine, engine::general_purpose::STANDARD}; +use serde::{Deserialize, Serialize}; + +use crate::{OpenAiError, OpenAiResult, Usage}; + +const DEFAULT_ENCODING_FORMAT: &str = "float"; + +/// OpenAI-compatible embedding input. Token arrays are preserved so clients +/// can avoid a second tokenizer pass when they already own tokenization. +#[derive(Debug, Clone, Deserialize, PartialEq)] +#[serde(untagged)] +pub enum EmbeddingInput { + Text(String), + Texts(Vec), + Tokens(Vec), + TokenArrays(Vec>), +} + +impl EmbeddingInput { + pub fn len(&self) -> usize { + match self { + Self::Text(_) | Self::Tokens(_) => 1, + Self::Texts(values) => values.len(), + Self::TokenArrays(values) => values.len(), + } + } + + pub fn is_empty(&self) -> bool { + match self { + Self::Text(value) => value.is_empty(), + Self::Texts(values) => values.is_empty() || values.iter().any(String::is_empty), + Self::Tokens(values) => values.is_empty(), + Self::TokenArrays(values) => values.is_empty() || values.iter().any(Vec::is_empty), + } + } + + fn contains_invalid_token(&self) -> bool { + match self { + Self::Text(_) | Self::Texts(_) => false, + Self::Tokens(tokens) => tokens.iter().any(|token| *token < 0), + Self::TokenArrays(inputs) => inputs.iter().flatten().any(|token| *token < 0), + } + } +} + +#[derive(Debug, Clone, Deserialize, PartialEq)] +pub struct EmbeddingsRequest { + pub model: String, + pub input: EmbeddingInput, + #[serde(default = "default_encoding_format")] + pub encoding_format: String, + #[serde(default)] + pub dimensions: Option, + #[serde(default)] + pub user: Option, +} + +fn default_encoding_format() -> String { + DEFAULT_ENCODING_FORMAT.to_string() +} + +impl EmbeddingsRequest { + pub fn validate(&self) -> OpenAiResult<()> { + if self.model.trim().is_empty() { + return Err(OpenAiError::invalid_request("model must not be empty")); + } + if self.input.is_empty() { + return Err(OpenAiError::invalid_request( + "embedding input must contain at least one non-empty item", + )); + } + if self.input.contains_invalid_token() { + return Err(OpenAiError::invalid_request( + "embedding token IDs must be non-negative", + )); + } + if !matches!(self.encoding_format.as_str(), "float" | "base64") { + return Err(OpenAiError::invalid_request( + "encoding_format must be 'float' or 'base64'", + )); + } + if self.dimensions == Some(0) { + return Err(OpenAiError::invalid_request( + "dimensions must be greater than zero", + )); + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Embedding { + pub values: Vec, + pub index: usize, +} + +#[derive(Debug, Clone, Serialize, PartialEq)] +#[serde(untagged)] +pub enum EmbeddingOutput { + Float(Vec), + Base64(String), +} + +#[derive(Debug, Clone, Serialize, PartialEq)] +pub struct EmbeddingObject { + pub object: &'static str, + pub embedding: EmbeddingOutput, + pub index: usize, +} + +#[derive(Debug, Clone, Serialize, PartialEq)] +pub struct EmbeddingResponse { + pub object: &'static str, + pub data: Vec, + pub model: String, + pub usage: Usage, +} + +impl EmbeddingResponse { + pub fn from_embeddings( + model: String, + embeddings: Vec, + prompt_tokens: u32, + encoding_format: &str, + ) -> Self { + let data = embeddings + .into_iter() + .map(|embedding| EmbeddingObject { + object: "embedding", + embedding: if encoding_format == "base64" { + EmbeddingOutput::Base64(encode_f32_base64(&embedding.values)) + } else { + EmbeddingOutput::Float(embedding.values) + }, + index: embedding.index, + }) + .collect(); + Self { + object: "list", + data, + model, + usage: Usage { + prompt_tokens, + completion_tokens: 0, + total_tokens: prompt_tokens, + ..Usage::default() + }, + } + } +} + +fn encode_f32_base64(values: &[f32]) -> String { + let mut bytes = Vec::with_capacity(std::mem::size_of_val(values)); + for value in values { + bytes.extend_from_slice(&value.to_le_bytes()); + } + STANDARD.encode(bytes) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn base64_encoding_is_little_endian_f32() { + assert_eq!(encode_f32_base64(&[1.0, -2.0]), "AACAPwAAAMA="); + } + + #[test] + fn request_rejects_empty_batches_and_unknown_formats() { + let mut request = EmbeddingsRequest { + model: "embed".into(), + input: EmbeddingInput::Texts(Vec::new()), + encoding_format: "float".into(), + dimensions: None, + user: None, + }; + assert!(request.validate().is_err()); + request.input = EmbeddingInput::Text("hello".into()); + request.encoding_format = "hex".into(); + assert!(request.validate().is_err()); + } + + #[test] + fn request_rejects_negative_token_ids() { + let request = EmbeddingsRequest { + model: "embed".into(), + input: EmbeddingInput::TokenArrays(vec![vec![1, 2], vec![3, -1]]), + encoding_format: "float".into(), + dimensions: None, + user: None, + }; + + let error = request.validate().expect_err("negative IDs must fail"); + assert_eq!(error.status(), axum::http::StatusCode::BAD_REQUEST); + assert!(error.body().error.message.contains("non-negative")); + } +} diff --git a/crates/openai-frontend/src/guardrails/compact.rs b/crates/openai-frontend/src/guardrails/compact.rs index 57c8b337bc..1f183b76df 100644 --- a/crates/openai-frontend/src/guardrails/compact.rs +++ b/crates/openai-frontend/src/guardrails/compact.rs @@ -6,13 +6,18 @@ use mesh_llm_guardrails::{ }; use crate::{ + audio::{ + AudioResponse, AudioSpeechRequest, AudioTranscriptionRequest, AudioTranscriptionResponse, + }, backend::{ ChatCompletionStream, CompletionStream, OpenAiBackend, OpenAiRequestContext, OpenAiResult, }, chat::{ChatCompletionRequest, ChatCompletionResponse}, completions::{CompletionRequest, CompletionResponse}, + embeddings::{EmbeddingResponse, EmbeddingsRequest}, errors::OpenAiError, models::ModelObject, + rerank::{RerankRequest, RerankResponse}, }; pub struct CompactingOpenAiBackend { @@ -108,4 +113,44 @@ impl OpenAiBackend for CompactingOpenAiBackend { ) -> OpenAiResult { self.backend.completion_stream(request, context).await } + + async fn embeddings( + &self, + request: EmbeddingsRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.embeddings(request, context).await + } + + async fn rerank( + &self, + request: RerankRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.rerank(request, context).await + } + + async fn audio_speech( + &self, + request: AudioSpeechRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.audio_speech(request, context).await + } + + async fn audio_transcription( + &self, + request: AudioTranscriptionRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.audio_transcription(request, context).await + } + + async fn audio_translation( + &self, + request: AudioTranscriptionRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.audio_translation(request, context).await + } } diff --git a/crates/openai-frontend/src/guardrails/mod.rs b/crates/openai-frontend/src/guardrails/mod.rs index 09a4924fc4..71fe4a0377 100644 --- a/crates/openai-frontend/src/guardrails/mod.rs +++ b/crates/openai-frontend/src/guardrails/mod.rs @@ -3,12 +3,17 @@ use std::sync::Arc; use async_trait::async_trait; use crate::{ + audio::{ + AudioResponse, AudioSpeechRequest, AudioTranscriptionRequest, AudioTranscriptionResponse, + }, backend::{ ChatCompletionStream, CompletionStream, OpenAiBackend, OpenAiRequestContext, OpenAiResult, }, chat::{ChatCompletionRequest, ChatCompletionResponse}, completions::{CompletionRequest, CompletionResponse}, + embeddings::{EmbeddingResponse, EmbeddingsRequest}, models::ModelObject, + rerank::{RerankRequest, RerankResponse}, }; mod compact; @@ -350,6 +355,46 @@ impl OpenAiBackend for GuardedOpenAiBackend { ) -> OpenAiResult { self.backend.completion_stream(request, context).await } + + async fn embeddings( + &self, + request: EmbeddingsRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.embeddings(request, context).await + } + + async fn rerank( + &self, + request: RerankRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.rerank(request, context).await + } + + async fn audio_speech( + &self, + request: AudioSpeechRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.audio_speech(request, context).await + } + + async fn audio_transcription( + &self, + request: AudioTranscriptionRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.audio_transcription(request, context).await + } + + async fn audio_translation( + &self, + request: AudioTranscriptionRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.audio_translation(request, context).await + } } #[cfg(test)] diff --git a/crates/openai-frontend/src/hooks.rs b/crates/openai-frontend/src/hooks.rs index b78914f28b..6444a0d51b 100644 --- a/crates/openai-frontend/src/hooks.rs +++ b/crates/openai-frontend/src/hooks.rs @@ -4,6 +4,9 @@ use async_trait::async_trait; use serde_json::Value; use crate::{ + audio::{ + AudioResponse, AudioSpeechRequest, AudioTranscriptionRequest, AudioTranscriptionResponse, + }, backend::{ ChatCompletionStream, CompletionStream, OpenAiBackend, OpenAiRequestContext, OpenAiResult, }, @@ -12,7 +15,9 @@ use crate::{ ChatMessage, MessageContent, MessageContentPart, capsule_id_is_valid, }, completions::{CompletionRequest, CompletionResponse}, + embeddings::{EmbeddingResponse, EmbeddingsRequest}, models::ModelObject, + rerank::{RerankRequest, RerankResponse}, }; pub const MESH_HOOKS_FIELD: &str = "mesh_hooks"; @@ -377,7 +382,7 @@ impl Drop for TerminalGuard { /// - A chunk carrying an error fires [`ChatCompletionOutcome::Error`] /// immediately — matching the non-streaming path, which never waits for a /// graceful end once the backend has already reported failure. -/// - Both fire via [`TerminalGuard::fire_detached`], since neither can +/// - Both fire via `TerminalGuard::fire_detached`, since neither can /// `.await` inside `poll_next`. /// - Dropping this wrapper before either of the above happens — an outer /// timeout, or the client disconnecting mid-stream — drops the @@ -609,6 +614,46 @@ impl OpenAiBackend for HookedOpenAiBackend { ) -> OpenAiResult { self.backend.completion_stream(request, context).await } + + async fn embeddings( + &self, + request: EmbeddingsRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.embeddings(request, context).await + } + + async fn rerank( + &self, + request: RerankRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.rerank(request, context).await + } + + async fn audio_speech( + &self, + request: AudioSpeechRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.audio_speech(request, context).await + } + + async fn audio_transcription( + &self, + request: AudioTranscriptionRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.audio_transcription(request, context).await + } + + async fn audio_translation( + &self, + request: AudioTranscriptionRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.audio_translation(request, context).await + } } pub fn chat_mesh_hooks_enabled(request: &ChatCompletionRequest) -> bool { diff --git a/crates/openai-frontend/src/lib.rs b/crates/openai-frontend/src/lib.rs index a6a032080e..6f88ef3eb5 100644 --- a/crates/openai-frontend/src/lib.rs +++ b/crates/openai-frontend/src/lib.rs @@ -1,19 +1,26 @@ +pub mod audio; pub mod backend; mod backend_lifecycle; pub mod chat; pub mod common; pub mod completions; +pub mod embeddings; pub mod errors; mod guardrails; pub mod hooks; pub mod lifecycle; pub mod models; mod request_lifecycle; +pub mod rerank; pub mod responses; pub mod router; pub mod sse; mod stream_lifecycle; +pub use audio::{ + AudioFormat, AudioResponse, AudioSpeechRequest, AudioTranscriptionRequest, + AudioTranscriptionResponse, +}; pub use backend::{ CancellationToken, ChatCompletionStream, CompletionStream, OpenAiBackend, OpenAiRequestContext, OpenAiResult, @@ -34,6 +41,9 @@ pub use completions::{ CompletionChoice, CompletionChunk, CompletionChunkChoice, CompletionPrompt, CompletionRequest, CompletionResponse, }; +pub use embeddings::{ + Embedding, EmbeddingInput, EmbeddingOutput, EmbeddingResponse, EmbeddingsRequest, +}; pub use errors::{OpenAiError, OpenAiErrorKind, already_openai_error, map_upstream_error_body}; pub use guardrails::{ CompactingOpenAiBackend, CompactionConfig, CompactionDecision, CompactionOverride, @@ -56,6 +66,7 @@ pub use lifecycle::{ parse_single_request_id, request_id_from_headers_or_generate, request_id_response_header, }; pub use models::{ModelId, ModelIdError, ModelObject, ModelsResponse}; +pub use rerank::{RerankDocument, RerankRequest, RerankResponse, RerankResult}; pub use responses::{ NormalizationOutcome, ResponseAdapterMode, ResponsesRequest, StreamUsage, chat_usage_to_responses_usage, normalize_openai_compat_request, parse_chat_stream_chunk, diff --git a/crates/openai-frontend/src/lifecycle.rs b/crates/openai-frontend/src/lifecycle.rs index 47159f910f..b072531e77 100644 --- a/crates/openai-frontend/src/lifecycle.rs +++ b/crates/openai-frontend/src/lifecycle.rs @@ -73,6 +73,11 @@ pub enum OpenAiFrontendRoute { Healthz, Readyz, Models, + Embeddings, + Rerank, + AudioSpeech, + AudioTranscriptions, + AudioTranslations, ChatCompletions, Completions, Responses, @@ -83,6 +88,11 @@ pub enum OpenAiFrontendRoute { #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum OpenAiBackendOperation { Models, + Embeddings, + Rerank, + AudioSpeech, + AudioTranscription, + AudioTranslation, ChatCompletion, ChatCompletionStream, Completion, diff --git a/crates/openai-frontend/src/rerank.rs b/crates/openai-frontend/src/rerank.rs new file mode 100644 index 0000000000..b859493ad1 --- /dev/null +++ b/crates/openai-frontend/src/rerank.rs @@ -0,0 +1,92 @@ +use serde::{Deserialize, Serialize}; + +use crate::{OpenAiError, OpenAiResult, Usage}; + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] +#[serde(untagged)] +pub enum RerankDocument { + Text(String), + Object(serde_json::Value), +} + +impl RerankDocument { + pub fn text(&self) -> OpenAiResult<&str> { + match self { + Self::Text(text) => Ok(text), + Self::Object(value) => value + .get("text") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| { + OpenAiError::invalid_request( + "rerank document objects must contain a string 'text' field", + ) + }), + } + } +} + +#[derive(Debug, Clone, Deserialize, PartialEq)] +pub struct RerankRequest { + pub model: String, + pub query: String, + pub documents: Vec, + #[serde(default)] + pub top_n: Option, + #[serde(default)] + pub return_documents: bool, +} + +impl RerankRequest { + pub fn validate(&self) -> OpenAiResult<()> { + if self.model.trim().is_empty() { + return Err(OpenAiError::invalid_request("model must not be empty")); + } + if self.query.is_empty() { + return Err(OpenAiError::invalid_request("query must not be empty")); + } + if self.documents.is_empty() { + return Err(OpenAiError::invalid_request( + "documents must contain at least one item", + )); + } + for document in &self.documents { + if document.text()?.is_empty() { + return Err(OpenAiError::invalid_request( + "rerank documents must not be empty", + )); + } + } + if self.top_n == Some(0) { + return Err(OpenAiError::invalid_request( + "top_n must be greater than zero", + )); + } + Ok(()) + } +} + +#[derive(Debug, Clone, Serialize, PartialEq)] +pub struct RerankResult { + pub index: usize, + pub relevance_score: f32, + #[serde(skip_serializing_if = "Option::is_none")] + pub document: Option, +} + +#[derive(Debug, Clone, Serialize, PartialEq)] +pub struct RerankResponse { + pub id: String, + pub results: Vec, + pub usage: Usage, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn document_objects_require_text() { + let document = RerankDocument::Object(serde_json::json!({"title": "missing"})); + assert!(document.text().is_err()); + } +} diff --git a/crates/openai-frontend/src/router.rs b/crates/openai-frontend/src/router.rs index 56846b40e3..b278385ee9 100644 --- a/crates/openai-frontend/src/router.rs +++ b/crates/openai-frontend/src/router.rs @@ -7,8 +7,15 @@ use std::{ use axum::{ Json, Router, body::Body, - extract::{DefaultBodyLimit, Extension, State, rejection::JsonRejection}, - http::{HeaderMap, HeaderValue, Method, Request, StatusCode, Uri, header::HeaderName}, + extract::{ + DefaultBodyLimit, Extension, Multipart, State, + multipart::{MultipartError, MultipartRejection}, + rejection::JsonRejection, + }, + http::{ + HeaderMap, HeaderValue, Method, Request, StatusCode, Uri, + header::{self, HeaderName}, + }, middleware::{self, Next}, response::{IntoResponse, Response, sse::Event}, routing::{get, post}, @@ -19,11 +26,15 @@ use serde::Serialize; use serde_json::Value; use crate::{ + audio::{ + AudioResponse, AudioSpeechRequest, AudioTranscriptionRequest, AudioTranscriptionResponse, + }, backend::{OpenAiBackend, OpenAiRequestContext, OpenAiResult, SharedBackend}, backend_lifecycle::{call_backend, call_backend_with_context}, chat::{CapsuleMarker, ChatCompletionChunk, ChatCompletionRequest}, common::{AgentSessionIdentity, AgentSessionSource, Usage}, completions::CompletionRequest, + embeddings::{EmbeddingResponse, EmbeddingsRequest}, errors::OpenAiError, lifecycle::{ CLIENT_NONCE_HEADER, CLIENT_NONCE_ORIGIN_HEADER, OpenAiBackendOperation, @@ -33,6 +44,7 @@ use crate::{ }, models::ModelsResponse, request_lifecycle::RequestLifecycle, + rerank::{RerankRequest, RerankResponse}, responses::{ ResponseAdapterMode, ResponseSseState, chunk_delta_text, normalize_openai_compat_request, responses_stream_completed_event_with_sequence, responses_stream_content_part_added_event, @@ -50,6 +62,7 @@ use crate::{ const AGENT_SESSION_HEADER_ENV: &str = "MESH_AGENT_SESSION_HEADER"; const BACKEND_TIMEOUT_SECS_ENV: &str = "MESH_OPENAI_BACKEND_TIMEOUT_SECS"; +const MAX_AUDIO_MULTIPART_BODY_BYTES: usize = 64 * 1024 * 1024 + 1024 * 1024; /// Backend timeout override, in whole seconds. `0` disables the timeout. /// @@ -255,6 +268,17 @@ pub fn router_for_with_config( .route("/healthz", get(health)) .route("/readyz", get(ready)) .route("/v1/models", get(models)) + .route("/v1/embeddings", post(embeddings)) + .route("/v1/rerank", post(rerank)) + .route("/v1/audio/speech", post(audio_speech)) + .route( + "/v1/audio/transcriptions", + post(audio_transcriptions).layer(DefaultBodyLimit::max(MAX_AUDIO_MULTIPART_BODY_BYTES)), + ) + .route( + "/v1/audio/translations", + post(audio_translations).layer(DefaultBodyLimit::max(MAX_AUDIO_MULTIPART_BODY_BYTES)), + ) .route("/v1/chat/completions", post(chat_completions)) .route("/v1/completions", post(completions)) .route("/v1/responses", post(responses)) @@ -312,6 +336,226 @@ async fn models( })) } +async fn embeddings( + State(state): State, + Extension(context): Extension, + payload: Result, JsonRejection>, +) -> Result { + let Json(request) = json_payload(payload)?; + request.validate()?; + let backend_context = OpenAiRequestContext::with_request_id(context.request_id); + let response: EmbeddingResponse = call_backend_with_context( + state.config.lifecycle_observer.clone(), + &context, + OpenAiBackendOperation::Embeddings, + "embeddings", + state.config.backend_timeout, + &backend_context, + state.backend.embeddings(request, backend_context.clone()), + ) + .await?; + state.response_completed( + &context, + OpenAiBackendOperation::Embeddings, + &response.usage, + ); + let usage = response.usage.clone(); + Ok(json_response_with_usage(response, &usage)) +} + +async fn rerank( + State(state): State, + Extension(context): Extension, + payload: Result, JsonRejection>, +) -> Result { + let Json(request) = json_payload(payload)?; + request.validate()?; + let backend_context = OpenAiRequestContext::with_request_id(context.request_id); + let response: RerankResponse = call_backend_with_context( + state.config.lifecycle_observer.clone(), + &context, + OpenAiBackendOperation::Rerank, + "rerank", + state.config.backend_timeout, + &backend_context, + state.backend.rerank(request, backend_context.clone()), + ) + .await?; + state.response_completed(&context, OpenAiBackendOperation::Rerank, &response.usage); + let usage = response.usage.clone(); + Ok(json_response_with_usage(response, &usage)) +} + +async fn audio_speech( + State(state): State, + Extension(context): Extension, + payload: Result, JsonRejection>, +) -> Result { + let Json(request) = json_payload(payload)?; + request.validate()?; + let backend_context = OpenAiRequestContext::with_request_id(context.request_id); + let response = call_backend_with_context( + state.config.lifecycle_observer.clone(), + &context, + OpenAiBackendOperation::AudioSpeech, + "audio_speech", + state.config.backend_timeout, + &backend_context, + state.backend.audio_speech(request, backend_context.clone()), + ) + .await?; + audio_response(response) +} + +fn audio_response(audio: AudioResponse) -> Result { + let content_type = HeaderValue::from_str(&audio.content_type) + .map_err(|_| OpenAiError::backend("audio backend returned an invalid content type"))?; + let mut response = Response::new(Body::from(audio.bytes)); + response + .headers_mut() + .insert(header::CONTENT_TYPE, content_type); + Ok(response) +} + +async fn audio_transcriptions( + State(state): State, + Extension(context): Extension, + multipart: Result, +) -> Result { + audio_text_request(state, context, multipart_payload(multipart)?, false).await +} + +async fn audio_translations( + State(state): State, + Extension(context): Extension, + multipart: Result, +) -> Result { + audio_text_request(state, context, multipart_payload(multipart)?, true).await +} + +fn multipart_payload(multipart: Result) -> OpenAiResult { + multipart.map_err(|error| { + OpenAiError::invalid_request(format!("invalid multipart request: {error}")) + }) +} + +fn multipart_error(error: MultipartError, field: &str) -> OpenAiError { + if error.status() == StatusCode::PAYLOAD_TOO_LARGE { + OpenAiError::payload_too_large(format!("{field} is too large: {error}")) + } else { + OpenAiError::invalid_request(format!("invalid {field}: {error}")) + } +} + +async fn audio_text_request( + state: FrontendState, + context: OpenAiLifecycleContext, + multipart: Multipart, + translate: bool, +) -> Result { + let request = parse_audio_multipart(multipart).await?; + request.validate()?; + let response_format = request.response_format.clone(); + let backend_context = OpenAiRequestContext::with_request_id(context.request_id); + let operation = if translate { + OpenAiBackendOperation::AudioTranslation + } else { + OpenAiBackendOperation::AudioTranscription + }; + let response: AudioTranscriptionResponse = call_backend_with_context( + state.config.lifecycle_observer.clone(), + &context, + operation, + if translate { + "audio_translation" + } else { + "audio_transcription" + }, + state.config.backend_timeout, + &backend_context, + if translate { + state + .backend + .audio_translation(request, backend_context.clone()) + } else { + state + .backend + .audio_transcription(request, backend_context.clone()) + }, + ) + .await?; + if response_format == "text" { + Ok(( + [(header::CONTENT_TYPE, "text/plain; charset=utf-8")], + response.text, + ) + .into_response()) + } else { + Ok(Json(response).into_response()) + } +} + +async fn parse_audio_multipart( + mut multipart: Multipart, +) -> OpenAiResult { + let mut model = None; + let mut file = None; + let mut filename = None; + let mut language = None; + let mut prompt = None; + let mut response_format = None; + let mut temperature = None; + + while let Some(field) = multipart + .next_field() + .await + .map_err(|error| multipart_error(error, "multipart body"))? + { + let name = field.name().unwrap_or_default().to_string(); + if name == "file" { + filename = field.file_name().map(str::to_owned); + let bytes = field + .bytes() + .await + .map_err(|error| multipart_error(error, "audio file field"))?; + file = Some(bytes.to_vec()); + continue; + } + if name == "model" && model.is_some() { + return Err(OpenAiError::invalid_request( + "duplicate multipart model field", + )); + } + let value = field + .text() + .await + .map_err(|error| multipart_error(error, "multipart text field"))?; + match name.as_str() { + "model" => model = Some(value), + "language" => language = Some(value), + "prompt" => prompt = Some(value), + "response_format" => response_format = Some(value), + "temperature" => { + temperature = + Some(value.parse::().map_err(|_| { + OpenAiError::invalid_request("temperature must be a number") + })?); + } + _ => {} + } + } + + Ok(AudioTranscriptionRequest { + model: model.ok_or_else(|| OpenAiError::invalid_request("model field is required"))?, + file: file.ok_or_else(|| OpenAiError::invalid_request("file field is required"))?, + filename, + language, + prompt, + response_format: response_format.unwrap_or_else(|| "json".to_string()), + temperature, + }) +} + async fn chat_completions( State(state): State, Extension(context): Extension, @@ -974,6 +1218,11 @@ fn lifecycle_route(uri: &Uri) -> OpenAiFrontendRoute { "/healthz" => OpenAiFrontendRoute::Healthz, "/readyz" => OpenAiFrontendRoute::Readyz, "/v1/models" => OpenAiFrontendRoute::Models, + "/v1/embeddings" => OpenAiFrontendRoute::Embeddings, + "/v1/rerank" => OpenAiFrontendRoute::Rerank, + "/v1/audio/speech" => OpenAiFrontendRoute::AudioSpeech, + "/v1/audio/transcriptions" => OpenAiFrontendRoute::AudioTranscriptions, + "/v1/audio/translations" => OpenAiFrontendRoute::AudioTranslations, "/v1/chat/completions" => OpenAiFrontendRoute::ChatCompletions, "/v1/completions" => OpenAiFrontendRoute::Completions, "/v1/responses" => OpenAiFrontendRoute::Responses, diff --git a/crates/openai-frontend/src/router_tests.rs b/crates/openai-frontend/src/router_tests.rs index 07ef28388e..6218cffbca 100644 --- a/crates/openai-frontend/src/router_tests.rs +++ b/crates/openai-frontend/src/router_tests.rs @@ -565,6 +565,80 @@ impl OpenAiBackend for FakeBackend { )), ]))) } + + async fn embeddings( + &self, + request: EmbeddingsRequest, + _context: OpenAiRequestContext, + ) -> OpenAiResult { + let embeddings = (0..request.input.len()) + .map(|index| crate::Embedding { + values: vec![index as f32 + 1.0, -0.5], + index, + }) + .collect(); + Ok(EmbeddingResponse::from_embeddings( + request.model, + embeddings, + 7, + &request.encoding_format, + )) + } + + async fn rerank( + &self, + request: RerankRequest, + _context: OpenAiRequestContext, + ) -> OpenAiResult { + let mut results = request + .documents + .iter() + .enumerate() + .map(|(index, document)| crate::RerankResult { + index, + relevance_score: (index + 1) as f32 / 10.0, + document: request.return_documents.then(|| document.clone()), + }) + .collect::>(); + results.sort_by(|left, right| right.relevance_score.total_cmp(&left.relevance_score)); + results.truncate(request.top_n.unwrap_or(results.len())); + Ok(RerankResponse { + id: "rerank_test".to_string(), + results, + usage: Usage::new(11, 0), + }) + } + + async fn audio_speech( + &self, + request: AudioSpeechRequest, + _context: OpenAiRequestContext, + ) -> OpenAiResult { + AudioResponse::new( + vec![0x52, 0x49, 0x46, 0x46], + request.response_format.content_type(), + ) + } + + async fn audio_transcription( + &self, + request: AudioTranscriptionRequest, + _context: OpenAiRequestContext, + ) -> OpenAiResult { + Ok(AudioTranscriptionResponse { + text: format!("transcribed {} bytes", request.file.len()), + }) + } + + async fn audio_translation( + &self, + request: AudioTranscriptionRequest, + _context: OpenAiRequestContext, + ) -> OpenAiResult { + Ok(AudioTranscriptionResponse { + text: format!("translated {} bytes", request.file.len()), + }) + } } struct SlowBackend; @@ -775,6 +849,9 @@ async fn models_route_returns_model_list() { assert_eq!(body["data"][0]["id"], "org/repo:Q4_K_M"); } +#[path = "router_tests/non_chat.rs"] +mod non_chat; + #[tokio::test] async fn health_route_returns_liveness_probe() { let app = router_for(Arc::new(FakeBackend)); diff --git a/crates/openai-frontend/src/router_tests/non_chat.rs b/crates/openai-frontend/src/router_tests/non_chat.rs new file mode 100644 index 0000000000..08766e77e7 --- /dev/null +++ b/crates/openai-frontend/src/router_tests/non_chat.rs @@ -0,0 +1,223 @@ +use super::*; + +#[tokio::test] +async fn embeddings_route_preserves_batch_order_and_usage() { + let response = post_json( + "/v1/embeddings", + json!({ + "model": "embed-model", + "input": ["first", "second"], + "encoding_format": "float" + }), + ) + .await; + + assert_eq!(response.status(), StatusCode::OK); + let body = response_body_json(response).await; + assert_eq!(body["object"], "list"); + assert_eq!(body["model"], "embed-model"); + assert_eq!(body["data"][0]["index"], 0); + assert_eq!(body["data"][0]["embedding"], json!([1.0, -0.5])); + assert_eq!(body["data"][1]["index"], 1); + assert_eq!(body["data"][1]["embedding"], json!([2.0, -0.5])); + assert_eq!(body["usage"]["prompt_tokens"], 7); + assert_eq!(body["usage"]["total_tokens"], 7); +} + +#[tokio::test] +async fn embeddings_route_returns_openai_error_for_invalid_format() { + let response = post_json( + "/v1/embeddings", + json!({ + "model": "embed-model", + "input": "hello", + "encoding_format": "hex" + }), + ) + .await; + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let body = response_body_json(response).await; + assert_eq!(body["error"]["type"], "invalid_request_error"); + assert_eq!(body["error"]["code"], "invalid_value"); +} + +#[tokio::test] +async fn rerank_route_sorts_limits_and_optionally_returns_documents() { + let response = post_json( + "/v1/rerank", + json!({ + "model": "rerank-model", + "query": "query", + "documents": ["one", {"text": "two", "title": "Two"}], + "top_n": 1, + "return_documents": true + }), + ) + .await; + + assert_eq!(response.status(), StatusCode::OK); + let body = response_body_json(response).await; + assert_eq!(body["id"], "rerank_test"); + assert_eq!(body["results"].as_array().unwrap().len(), 1); + assert_eq!(body["results"][0]["index"], 1); + assert_eq!(body["results"][0]["relevance_score"], 0.2); + assert_eq!(body["results"][0]["document"]["text"], "two"); + assert_eq!(body["usage"]["prompt_tokens"], 11); +} + +#[tokio::test] +async fn audio_speech_route_returns_backend_bytes_and_content_type() { + let response = post_json( + "/v1/audio/speech", + json!({ + "model": "tts-model", + "input": "Hello", + "voice": "default", + "response_format": "wav" + }), + ) + .await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers()["content-type"], "audio/wav"); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(bytes.as_ref(), b"RIFF"); +} + +fn audio_multipart(boundary: &str, response_format: &str) -> Vec { + format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\naudio-model\r\n\ + --{boundary}\r\nContent-Disposition: form-data; name=\"response_format\"\r\n\r\n{response_format}\r\n\ + --{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"sample.wav\"\r\n\ + Content-Type: audio/wav\r\n\r\nWAVE\r\n--{boundary}--\r\n" + ) + .into_bytes() +} + +async fn post_audio_multipart(path: &str, boundary: &str, body: Vec) -> Response { + router_for(Arc::new(FakeBackend)) + .oneshot( + Request::builder() + .method("POST") + .uri(path) + .header( + "content-type", + format!("multipart/form-data; boundary={boundary}"), + ) + .body(Body::from(body)) + .unwrap(), + ) + .await + .unwrap() +} + +#[tokio::test] +async fn audio_transcription_supports_json_and_text_responses() { + let boundary = "mesh-audio-boundary"; + let json_response = post_audio_multipart( + "/v1/audio/transcriptions", + boundary, + audio_multipart(boundary, "json"), + ) + .await; + assert_eq!(json_response.status(), StatusCode::OK); + let body = response_body_json(json_response).await; + assert_eq!(body["text"], "transcribed 4 bytes"); + + let text_response = post_audio_multipart( + "/v1/audio/transcriptions", + boundary, + audio_multipart(boundary, "text"), + ) + .await; + assert_eq!(text_response.status(), StatusCode::OK); + assert_eq!( + text_response.headers()["content-type"], + "text/plain; charset=utf-8" + ); + assert_eq!( + response_body_text(text_response).await, + "transcribed 4 bytes" + ); +} + +#[tokio::test] +async fn audio_translation_uses_translation_backend() { + let boundary = "mesh-audio-boundary"; + let response = post_audio_multipart( + "/v1/audio/translations", + boundary, + audio_multipart(boundary, "json"), + ) + .await; + + assert_eq!(response.status(), StatusCode::OK); + let body = response_body_json(response).await; + assert_eq!(body["text"], "translated 4 bytes"); +} + +#[tokio::test] +async fn audio_upload_uses_its_dedicated_body_limit() { + let boundary = "mesh-large-audio-boundary"; + let file_bytes = vec![0x2a; OpenAiFrontendConfig::default().max_request_body_bytes + 1]; + let mut body = format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\naudio-model\r\n\ + --{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"sample.wav\"\r\n\ + Content-Type: audio/wav\r\n\r\n" + ) + .into_bytes(); + body.extend_from_slice(&file_bytes); + body.extend_from_slice(format!("\r\n--{boundary}--\r\n").as_bytes()); + + let response = post_audio_multipart("/v1/audio/transcriptions", boundary, body).await; + + assert_eq!(response.status(), StatusCode::OK); + let response = response_body_json(response).await; + assert_eq!( + response["text"], + format!("transcribed {} bytes", file_bytes.len()) + ); +} + +#[tokio::test] +async fn malformed_audio_multipart_uses_openai_error_envelope() { + let response = router_for(Arc::new(FakeBackend)) + .oneshot( + Request::builder() + .method("POST") + .uri("/v1/audio/transcriptions") + .header("content-type", "multipart/form-data") + .body(Body::from("not multipart")) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let body = response_body_json(response).await; + assert_eq!(body["error"]["type"], "invalid_request_error"); +} + +#[tokio::test] +async fn duplicate_audio_model_field_is_rejected() { + let boundary = "mesh-audio-boundary"; + let body = format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\naudio-model\r\n\ + --{boundary}\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\nother-model\r\n\ + --{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"sample.wav\"\r\n\ + Content-Type: audio/wav\r\n\r\nWAVE\r\n--{boundary}--\r\n" + ); + for path in ["/v1/audio/transcriptions", "/v1/audio/translations"] { + let response = post_audio_multipart(path, boundary, body.as_bytes().to_vec()).await; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let payload = response_body_json(response).await; + assert_eq!(payload["error"]["type"], "invalid_request_error"); + assert!( + payload["error"]["message"] + .as_str() + .unwrap() + .contains("duplicate multipart model field") + ); + } +} diff --git a/crates/skippy-ffi/build.rs b/crates/skippy-ffi/build.rs index d9e66567fc..08e9d78f77 100644 --- a/crates/skippy-ffi/build.rs +++ b/crates/skippy-ffi/build.rs @@ -167,24 +167,33 @@ fn main() { println!("cargo:rustc-link-lib=static=llama-common-base"); println!("cargo:rustc-link-lib=static=llama"); println!("cargo:rustc-link-lib=static=ggml"); - let has_cuda = static_archive_exists( + let has_cuda = configured_backend_archive( &build_dir, + &cmake_cache, + backend == "cuda", + "GGML_CUDA", "ggml/src/ggml-cuda/libggml-cuda.a", "ggml/src/ggml-cuda/ggml-cuda.lib", ); if has_cuda { println!("cargo:rustc-link-lib=static=ggml-cuda"); } - let has_hip = static_archive_exists( + let has_hip = configured_backend_archive( &build_dir, + &cmake_cache, + backend == "rocm" || backend == "hip", + "GGML_HIP", "ggml/src/ggml-hip/libggml-hip.a", "ggml/src/ggml-hip/ggml-hip.lib", ); if has_hip { println!("cargo:rustc-link-lib=static=ggml-hip"); } - let has_vulkan = static_archive_exists( + let has_vulkan = configured_backend_archive( &build_dir, + &cmake_cache, + backend == "vulkan", + "GGML_VULKAN", "ggml/src/ggml-vulkan/libggml-vulkan.a", "ggml/src/ggml-vulkan/ggml-vulkan.lib", ); @@ -192,18 +201,24 @@ fn main() { println!("cargo:rustc-link-lib=static=ggml-vulkan"); } println!("cargo:rustc-link-lib=static=ggml-cpu"); - if static_archive_exists( - &build_dir, - "ggml/src/ggml-blas/libggml-blas.a", - "ggml/src/ggml-blas/ggml-blas.lib", - ) { + if cmake_bool_enabled(&cmake_cache, "GGML_BLAS") + && static_archive_exists( + &build_dir, + "ggml/src/ggml-blas/libggml-blas.a", + "ggml/src/ggml-blas/ggml-blas.lib", + ) + { println!("cargo:rustc-link-lib=static=ggml-blas"); } - if static_archive_exists( + let has_metal = configured_backend_archive( &build_dir, + &cmake_cache, + backend == "metal", + "GGML_METAL", "ggml/src/ggml-metal/libggml-metal.a", "ggml/src/ggml-metal/ggml-metal.lib", - ) { + ); + if has_metal { println!("cargo:rustc-link-lib=static=ggml-metal"); } println!("cargo:rustc-link-lib=static=ggml-base"); @@ -212,11 +227,7 @@ fn main() { link_apple_openmp_libs(&cmake_cache); println!("cargo:rustc-link-lib=c++"); println!("cargo:rustc-link-lib=framework=Accelerate"); - if static_archive_exists( - &build_dir, - "ggml/src/ggml-metal/libggml-metal.a", - "ggml/src/ggml-metal/ggml-metal.lib", - ) { + if has_metal { println!("cargo:rustc-link-lib=framework=Foundation"); println!("cargo:rustc-link-lib=framework=Metal"); println!("cargo:rustc-link-lib=framework=MetalKit"); @@ -434,6 +445,41 @@ fn static_archive_exists( build_dir.join(unix_archive).exists() || build_dir.join(msvc_archive).exists() } +fn cmake_bool_enabled(cache: &std::path::Path, key: &str) -> bool { + let Ok(contents) = std::fs::read_to_string(cache) else { + return false; + }; + let prefix = format!("{key}:BOOL="); + contents + .lines() + .find_map(|line| line.strip_prefix(&prefix)) + .is_some_and(|value| matches!(value, "ON" | "TRUE" | "1")) +} + +fn configured_backend_archive( + build_dir: &std::path::Path, + cmake_cache: &std::path::Path, + selected_backend: bool, + cmake_key: &str, + unix_archive: &str, + msvc_archive: &str, +) -> bool { + if !selected_backend { + return false; + } + assert!( + cmake_bool_enabled(cmake_cache, cmake_key), + "selected backend requires {cmake_key}=ON in {}", + cmake_cache.display() + ); + assert!( + static_archive_exists(build_dir, unix_archive, msvc_archive), + "selected backend archive is missing from {}", + build_dir.display() + ); + true +} + fn link_linux_cuda_libs(cmake_cache: &std::path::Path) { for (cache_key, lib) in [ ("CUDA_cuda_driver_LIBRARY", "cuda"), diff --git a/crates/skippy-ffi/src/abi.rs b/crates/skippy-ffi/src/abi.rs index 895d0c11f5..8571b393c6 100644 --- a/crates/skippy-ffi/src/abi.rs +++ b/crates/skippy-ffi/src/abi.rs @@ -13,7 +13,63 @@ pub const FEATURE_INKLING_MTP_MM: u64 = 1 << 27; pub const FEATURE_ITERATION_BATCH: u64 = 1 << 28; pub const FEATURE_ACTIVATION_BOUNDARY: u64 = 1 << 29; pub const FEATURE_MODEL_SOURCE: u64 = 1 << 30; +pub const FEATURE_NON_CHAT_WORKLOADS: u64 = 1 << 31; pub const MODEL_TENSOR_SOURCE_V1_ABI_VERSION: u32 = 1; +pub const WORKLOAD_INFO_V1_ABI_VERSION: u32 = 1; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +#[repr(i32)] +pub enum WorkloadKind { + #[default] + CausalGeneration = 0, + Embedding = 1, + Rerank = 2, + EncoderDecoder = 3, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +#[repr(i32)] +pub enum WorkloadPooling { + Unspecified = -1, + #[default] + None = 0, + Mean = 1, + Cls = 2, + Last = 3, + Rank = 4, +} + +#[repr(C)] +#[derive(Debug, Clone, Copy)] +pub struct WorkloadInfoV1 { + pub abi_version: u32, + pub struct_size: u32, + pub kind: WorkloadKind, + pub pooling: WorkloadPooling, + pub output_dimensions: u32, + pub classifier_outputs: u32, + pub has_encoder: bool, + pub has_decoder: bool, + pub full_model_only: bool, + pub reserved0: u8, +} + +impl Default for WorkloadInfoV1 { + fn default() -> Self { + Self { + abi_version: WORKLOAD_INFO_V1_ABI_VERSION, + struct_size: std::mem::size_of::() as u32, + kind: WorkloadKind::default(), + pooling: WorkloadPooling::default(), + output_dimensions: 0, + classifier_outputs: 0, + has_encoder: false, + has_decoder: false, + full_model_only: true, + reserved0: 0, + } + } +} pub type ModelReadTensorF32Callback = Option< unsafe extern "C" fn( diff --git a/crates/skippy-ffi/src/dynamic.rs b/crates/skippy-ffi/src/dynamic.rs index 851990f52f..a524a8969f 100644 --- a/crates/skippy-ffi/src/dynamic.rs +++ b/crates/skippy-ffi/src/dynamic.rs @@ -8,13 +8,14 @@ use crate::{ ABI_VERSION_MAJOR, ABI_VERSION_MINOR, ABI_VERSION_PATCH, AbiVersion, ActivationBoundaryDesc, ActivationDesc, BackendDevice, Error, GenerationSignalWindow, IterationRequest, KvPageDesc, LlamaLogCallback, LlamaModelQuantizeParams, Model, ModelInfo, ModelTensorSourceV1, MtmdBitmap, - MtmdContext, MtmdContextParams, MtmdDecoderPos, MtmdHelperBitmapWrapper, MtmdHelperInitOpt, - MtmdHelperVideo, MtmdInputChunkType, MtmdInputChunks, MtmdInputText, NativeMtpDraft, - NativeRuntimeLoadError, NgramCache, Opaque, RuntimeConfig, SamplingConfig, Session, - SkippyDecodeStepSampledMtpFn, SkippyModelAttachMtpDraftModelFn, SkippyRuntimeEventReporterV1, - SlicePlan, StagePlan, StagePlanDescV1, StagePlanProfileDescV1, StagePlanStateDescV1, - StagePlanStringRefV1, StagePlanValueDescV1, StagePlanValueKind, StagePlanner, - StagePlannerConfigV1, Status, TensorInfo, TokenSignal, runtime_abi_supported, + MtmdContext, MtmdContextParams, MtmdDecoderPos, MtmdGenAudioInfo, MtmdHelperBitmapWrapper, + MtmdHelperGenAudio, MtmdHelperGenAudioInput, MtmdHelperInitOpt, MtmdHelperVideo, + MtmdInputChunkType, MtmdInputChunks, MtmdInputText, NativeMtpDraft, NativeRuntimeLoadError, + NgramCache, Opaque, RuntimeConfig, SamplingConfig, Session, SkippyDecodeStepSampledMtpFn, + SkippyModelAttachMtpDraftModelFn, SkippyRuntimeEventReporterV1, SlicePlan, StagePlan, + StagePlanDescV1, StagePlanProfileDescV1, StagePlanStateDescV1, StagePlanStringRefV1, + StagePlanValueDescV1, StagePlanValueKind, StagePlanner, StagePlannerConfigV1, Status, + TensorInfo, TokenSignal, WorkloadInfoV1, runtime_abi_supported, }; static SYMBOLS: OnceLock = OnceLock::new(); @@ -164,6 +165,8 @@ macro_rules! dynamic_symbols { } dynamic_symbols! { + llama_get_embeddings_ith(ctx: *mut Opaque, index: i32) -> *mut f32; + llama_set_embeddings(ctx: *mut Opaque, embeddings: bool); llama_log_set(log_callback: LlamaLogCallback, user_data: *mut c_void); ggml_log_set(log_callback: LlamaLogCallback, user_data: *mut c_void); llama_model_quantize_default_params() -> LlamaModelQuantizeParams; @@ -183,6 +186,7 @@ dynamic_symbols! { skippy_model_llama_model(model: *const Model) -> *const Opaque; skippy_model_output_activation_boundary(model: *const Model, out_desc: *mut ActivationBoundaryDesc) -> bool; skippy_model_input_activation_boundary(model: *const Model, out_desc: *mut ActivationBoundaryDesc) -> bool; + skippy_model_workload_info_v1(model: *const Model, out_info: *mut WorkloadInfoV1, out_error: *mut *mut Error) -> Status; skippy_session_create(model: *mut Model, out_session: *mut *mut Session, out_error: *mut *mut Error) -> Status; skippy_session_create_from_resident_prefix(model: *mut Model, cache_seq_id: i32, token_ids: *const i32, token_count: usize, out_session: *mut *mut Session, out_error: *mut *mut Error) -> Status; skippy_session_llama_context(session: *mut Session) -> *mut Opaque; @@ -194,6 +198,9 @@ dynamic_symbols! { skippy_session_set_position(session: *mut Session, n_past: i32, out_error: *mut *mut Error) -> Status; skippy_session_sample_current(session: *mut Session, sampling: *const SamplingConfig, out_predicted_token: *mut i32, out_error: *mut *mut Error) -> Status; skippy_session_configure_chat_sampling(session: *mut Session, sampling: *const SamplingConfig, metadata_json: *const c_char, prompt_token_count: u64, out_error: *mut *mut Error) -> Status; + skippy_session_embed(session: *mut Session, token_ids: *const i32, token_count: usize, output: *mut f32, output_capacity: usize, out_dimensions: *mut usize, out_error: *mut *mut Error) -> Status; + skippy_session_rerank(session: *mut Session, query: *const c_char, document: *const c_char, out_score: *mut f32, out_token_count: *mut usize, out_error: *mut *mut Error) -> Status; + skippy_session_encode_prompt(session: *mut Session, token_ids: *const i32, token_count: usize, out_decoder_start_token: *mut i32, out_error: *mut *mut Error) -> Status; skippy_session_reset(session: *mut Session, out_error: *mut *mut Error) -> Status; skippy_session_free(session: *mut Session, out_error: *mut *mut Error) -> Status; skippy_prefill_chunk(session: *mut Session, token_ids: *const i32, token_count: usize, input_activations: *const c_void, input_activation_bytes: usize, output_activations: *mut c_void, output_activation_capacity: usize, out_output_activation_bytes: *mut usize, out_error: *mut *mut Error) -> Status; @@ -253,6 +260,14 @@ dynamic_symbols! { mtmd_default_marker() -> *const c_char; mtmd_helper_log_set(log_callback: LlamaLogCallback, user_data: *mut c_void); mtmd_context_params_default() -> MtmdContextParams; + mtmd_gen_audio_get_info(ctx: *const MtmdContext) -> MtmdGenAudioInfo; + mtmd_helper_gen_audio_init(lctx: *mut Opaque, mctx: *mut MtmdContext) -> *mut MtmdHelperGenAudio; + mtmd_helper_gen_audio_free(ctx: *mut MtmdHelperGenAudio); + mtmd_helper_gen_audio_reset(ctx: *mut MtmdHelperGenAudio); + mtmd_helper_gen_audio_set_input(ctx: *mut MtmdHelperGenAudio, input: *const MtmdHelperGenAudioInput) -> i32; + mtmd_helper_gen_audio_step_prompt(ctx: *mut MtmdHelperGenAudio, n_batch: i32) -> i32; + mtmd_helper_gen_audio_step_gen(ctx: *mut MtmdHelperGenAudio, sampled: i32, h_state_in: *const f32, h_state_out: *mut *const f32, out_stop: *mut bool) -> i32; + mtmd_helper_gen_audio_get_output(ctx: *mut MtmdHelperGenAudio, out_sample_rate: *mut i32, out_data: *mut *const c_char, out_data_len: *mut usize, out_n_samples: *mut i64) -> i32; mtmd_init_from_file(mmproj_fname: *const c_char, text_model: *const Opaque, ctx_params: MtmdContextParams) -> *mut MtmdContext; mtmd_free(ctx: *mut MtmdContext); mtmd_helper_init_opt_default() -> MtmdHelperInitOpt; diff --git a/crates/skippy-ffi/src/lib.rs b/crates/skippy-ffi/src/lib.rs index 7b6253ed20..86ac75540e 100644 --- a/crates/skippy-ffi/src/lib.rs +++ b/crates/skippy-ffi/src/lib.rs @@ -5,7 +5,7 @@ mod dynamic_library; // without compiling the crate to determine native-runtime compatibility. pub const ABI_VERSION_MAJOR: u32 = 0; pub const ABI_VERSION_MINOR: u32 = 1; -pub const ABI_VERSION_PATCH: u32 = 53; +pub const ABI_VERSION_PATCH: u32 = 54; mod abi; mod activation; @@ -29,14 +29,16 @@ pub use abi::{ BACKEND_DEVICE_CAP_HOST_BUFFER, BackendDevice, BackendDeviceType, Error, FEATURE_ACTIVATION_BOUNDARY, FEATURE_BACKEND_DEVICES, FEATURE_INKLING_MTP_MM, FEATURE_ITERATION_BATCH, FEATURE_MODEL_SOURCE, FEATURE_NATIVE_MTP_N1, - FEATURE_NGRAM_CACHE_DRAFT, FEATURE_RUNTIME_EVENTS, FEATURE_STAGE_PLAN, IterationRequest, - LlamaLogCallback, LoadMode, MODEL_TENSOR_SOURCE_V1_ABI_VERSION, Model, ModelImatrixEntryV1, - ModelInfo, ModelReadTensorF32Callback, ModelTensorSourceV1, MtmdProgressCallback, MtpSource, - NgramCache, Opaque, RuntimeConfig, Session, SkippyDecodeStepSampledMtpFn, - SkippyModelAttachMtpDraftModelFn, SkippyRuntimeEventCallback, SkippyRuntimeEventCategory, - SkippyRuntimeEventEmitterKind, SkippyRuntimeEventFailureCode, SkippyRuntimeEventKind, - SkippyRuntimeEventProgressUnit, SkippyRuntimeEventReporterV1, SkippyRuntimeEventV1, SlicePlan, - Status, TRISTATE_AUTO, TRISTATE_FALSE, TRISTATE_TRUE, TensorRole, runtime_abi_supported, + FEATURE_NGRAM_CACHE_DRAFT, FEATURE_NON_CHAT_WORKLOADS, FEATURE_RUNTIME_EVENTS, + FEATURE_STAGE_PLAN, IterationRequest, LlamaLogCallback, LoadMode, + MODEL_TENSOR_SOURCE_V1_ABI_VERSION, Model, ModelImatrixEntryV1, ModelInfo, + ModelReadTensorF32Callback, ModelTensorSourceV1, MtmdProgressCallback, MtpSource, NgramCache, + Opaque, RuntimeConfig, Session, SkippyDecodeStepSampledMtpFn, SkippyModelAttachMtpDraftModelFn, + SkippyRuntimeEventCallback, SkippyRuntimeEventCategory, SkippyRuntimeEventEmitterKind, + SkippyRuntimeEventFailureCode, SkippyRuntimeEventKind, SkippyRuntimeEventProgressUnit, + SkippyRuntimeEventReporterV1, SkippyRuntimeEventV1, SlicePlan, Status, TRISTATE_AUTO, + TRISTATE_FALSE, TRISTATE_TRUE, TensorRole, WORKLOAD_INFO_V1_ABI_VERSION, WorkloadInfoV1, + WorkloadKind, WorkloadPooling, runtime_abi_supported, }; pub use activation::{ ACTIVATION_FLAG_GEMMA3N_ALTUP, ACTIVATION_FLAG_INKLING_MTP_EMBD, ACTIVATION_SIDEBAND_TOKEN_IDS, @@ -47,9 +49,10 @@ pub use model::{ LlamaModelKvOverrideValue, LlamaModelQuantizeParams, LlamaModelTensorOverride, }; pub use multimodal::{ - MtmdBitmap, MtmdContext, MtmdContextParams, MtmdDecoderPos, MtmdHelperBitmapWrapper, - MtmdHelperInitOpt, MtmdHelperVideo, MtmdHelperVideoInitParams, MtmdInputChunkType, - MtmdInputChunks, MtmdInputText, + MtmdBitmap, MtmdContext, MtmdContextParams, MtmdDecoderPos, MtmdGenAudioInfo, MtmdGenAudioType, + MtmdHelperBitmapWrapper, MtmdHelperGenAudio, MtmdHelperGenAudioInput, + MtmdHelperGenAudioOutputType, MtmdHelperInitOpt, MtmdHelperVideo, MtmdHelperVideoInitParams, + MtmdInputChunkType, MtmdInputChunks, MtmdInputText, }; pub use runtime::{ NativeRuntimeLoadError, abi_features, llama_model_is_diffusion, llama_model_is_hybrid, @@ -84,21 +87,24 @@ pub use runtime::skippy_abi_features; #[cfg(feature = "dynamic-runtime")] pub use dynamic::{ - ggml_log_set, llama_log_set, llama_model_quantize, llama_model_quantize_default_params, - load_native_runtime_libraries, load_native_runtime_library, mtmd_bitmap_free, - mtmd_context_params_default, mtmd_decode_use_mrope, mtmd_default_marker, mtmd_free, + ggml_log_set, llama_get_embeddings_ith, llama_log_set, llama_model_quantize, + llama_model_quantize_default_params, llama_set_embeddings, load_native_runtime_libraries, + load_native_runtime_library, mtmd_bitmap_free, mtmd_context_params_default, + mtmd_decode_use_mrope, mtmd_default_marker, mtmd_free, mtmd_gen_audio_get_info, mtmd_helper_bitmap_init_from_buf, mtmd_helper_eval_chunk_single, mtmd_helper_eval_chunks, - mtmd_helper_get_n_pos, mtmd_helper_get_n_tokens, mtmd_helper_image_get_decoder_pos, - mtmd_helper_init_opt_default, mtmd_helper_log_set, mtmd_helper_video_free, mtmd_init_from_file, - mtmd_input_chunk_get_n_tokens, mtmd_input_chunk_get_tokens_image, - mtmd_input_chunk_get_tokens_text, mtmd_input_chunk_get_type, mtmd_input_chunks_free, - mtmd_input_chunks_get, mtmd_input_chunks_init, mtmd_input_chunks_size, mtmd_tokenize, - native_runtime_loaded, skippy_abi_features_optional, skippy_apply_chat_template_json, - skippy_backend_device_at, skippy_backend_device_count, skippy_decode_batch_sampled, - skippy_decode_step_frame_batch_sampled, skippy_decode_step_frame_sampled, - skippy_decode_step_frame_sampled_mtp, skippy_decode_step_sampled, - skippy_decode_step_sampled_mtp, skippy_decode_step_sampled_mtp_fn, skippy_detokenize, - skippy_error_free, skippy_export_full_state, skippy_export_kv_page, + mtmd_helper_gen_audio_free, mtmd_helper_gen_audio_get_output, mtmd_helper_gen_audio_init, + mtmd_helper_gen_audio_reset, mtmd_helper_gen_audio_set_input, mtmd_helper_gen_audio_step_gen, + mtmd_helper_gen_audio_step_prompt, mtmd_helper_get_n_pos, mtmd_helper_get_n_tokens, + mtmd_helper_image_get_decoder_pos, mtmd_helper_init_opt_default, mtmd_helper_log_set, + mtmd_helper_video_free, mtmd_init_from_file, mtmd_input_chunk_get_n_tokens, + mtmd_input_chunk_get_tokens_image, mtmd_input_chunk_get_tokens_text, mtmd_input_chunk_get_type, + mtmd_input_chunks_free, mtmd_input_chunks_get, mtmd_input_chunks_init, mtmd_input_chunks_size, + mtmd_tokenize, native_runtime_loaded, skippy_abi_features_optional, + skippy_apply_chat_template_json, skippy_backend_device_at, skippy_backend_device_count, + skippy_decode_batch_sampled, skippy_decode_step_frame_batch_sampled, + skippy_decode_step_frame_sampled, skippy_decode_step_frame_sampled_mtp, + skippy_decode_step_sampled, skippy_decode_step_sampled_mtp, skippy_decode_step_sampled_mtp_fn, + skippy_detokenize, skippy_error_free, skippy_export_full_state, skippy_export_kv_page, skippy_export_recurrent_state, skippy_export_state, skippy_import_full_state, skippy_import_kv_page, skippy_import_recurrent_state, skippy_import_state, skippy_iteration_batch_sampled, skippy_model_attach_mtp_draft_model_fn, skippy_model_free, @@ -107,40 +113,45 @@ pub use dynamic::{ skippy_model_llama_model, skippy_model_open, skippy_model_open_from_parts, skippy_model_open_from_parts_with_events_fn, skippy_model_open_from_source, skippy_model_open_with_events_fn, skippy_model_output_activation_boundary, - skippy_ngram_cache_append, skippy_ngram_cache_create, skippy_ngram_cache_draft, - skippy_ngram_cache_free, skippy_ngram_cache_reset, skippy_parse_chat_response_json, - skippy_prefill_chunk, skippy_prefill_chunk_frame, skippy_prefill_chunk_frame_sampled, - skippy_prefill_chunk_frame_sampled_with_positions, skippy_prefill_chunk_frame_with_positions, - skippy_retire_verify_checkpoint, skippy_session_batch_size, - skippy_session_begin_external_decode, skippy_session_configure_chat_sampling, - skippy_session_copy_output_activation_frame, skippy_session_create, - skippy_session_create_from_resident_prefix, skippy_session_drop_sequence, + skippy_model_workload_info_v1, skippy_ngram_cache_append, skippy_ngram_cache_create, + skippy_ngram_cache_draft, skippy_ngram_cache_free, skippy_ngram_cache_reset, + skippy_parse_chat_response_json, skippy_prefill_chunk, skippy_prefill_chunk_frame, + skippy_prefill_chunk_frame_sampled, skippy_prefill_chunk_frame_sampled_with_positions, + skippy_prefill_chunk_frame_with_positions, skippy_retire_verify_checkpoint, + skippy_session_batch_size, skippy_session_begin_external_decode, + skippy_session_configure_chat_sampling, skippy_session_copy_output_activation_frame, + skippy_session_create, skippy_session_create_from_resident_prefix, + skippy_session_drop_sequence, skippy_session_embed, skippy_session_encode_prompt, skippy_session_end_external_decode, skippy_session_free, skippy_session_last_token_signal, skippy_session_llama_context, skippy_session_memory_used_cells, skippy_session_position, - skippy_session_reset, skippy_session_restore_prefix, skippy_session_sample_current, - skippy_session_save_prefix, skippy_session_sequence_id, skippy_session_set_position, - skippy_session_signal_window, skippy_slice_plan_add_layer_range, skippy_slice_plan_create, - skippy_slice_plan_free, skippy_stage_plan_describe_v1, skippy_stage_plan_free, - skippy_stage_plan_profile_at_v1, skippy_stage_plan_resident_tensor_at_v1, - skippy_stage_plan_state_at_v1, skippy_stage_plan_string_v1, - skippy_stage_plan_validate_chain_v1, skippy_stage_plan_value_at_v1, - skippy_stage_planner_create_v1, skippy_stage_planner_free, skippy_stage_planner_realize_v1, - skippy_token_is_eog, skippy_tokenize, skippy_trim_session, skippy_verify_tokens, - skippy_verify_tokens_frame_sampled, skippy_write_gguf_from_parts, + skippy_session_rerank, skippy_session_reset, skippy_session_restore_prefix, + skippy_session_sample_current, skippy_session_save_prefix, skippy_session_sequence_id, + skippy_session_set_position, skippy_session_signal_window, skippy_slice_plan_add_layer_range, + skippy_slice_plan_create, skippy_slice_plan_free, skippy_stage_plan_describe_v1, + skippy_stage_plan_free, skippy_stage_plan_profile_at_v1, + skippy_stage_plan_resident_tensor_at_v1, skippy_stage_plan_state_at_v1, + skippy_stage_plan_string_v1, skippy_stage_plan_validate_chain_v1, + skippy_stage_plan_value_at_v1, skippy_stage_planner_create_v1, skippy_stage_planner_free, + skippy_stage_planner_realize_v1, skippy_token_is_eog, skippy_tokenize, skippy_trim_session, + skippy_verify_tokens, skippy_verify_tokens_frame_sampled, skippy_write_gguf_from_parts, skippy_write_gguf_metadata_from_parts, skippy_write_slice_gguf, }; #[cfg(not(feature = "dynamic-runtime"))] pub use static_bindings::{ - ggml_log_set, llama_log_set, llama_model_quantize, llama_model_quantize_default_params, - mtmd_bitmap_free, mtmd_context_params_default, mtmd_decode_use_mrope, mtmd_default_marker, - mtmd_free, mtmd_helper_bitmap_init_from_buf, mtmd_helper_eval_chunk_single, - mtmd_helper_eval_chunks, mtmd_helper_get_n_pos, mtmd_helper_get_n_tokens, - mtmd_helper_image_get_decoder_pos, mtmd_helper_init_opt_default, mtmd_helper_log_set, - mtmd_helper_video_free, mtmd_init_from_file, mtmd_input_chunk_get_n_tokens, - mtmd_input_chunk_get_tokens_image, mtmd_input_chunk_get_tokens_text, mtmd_input_chunk_get_type, - mtmd_input_chunks_free, mtmd_input_chunks_get, mtmd_input_chunks_init, mtmd_input_chunks_size, - mtmd_tokenize, skippy_abi_features, skippy_apply_chat_template_json, skippy_backend_device_at, + ggml_log_set, llama_get_embeddings_ith, llama_log_set, llama_model_quantize, + llama_model_quantize_default_params, llama_set_embeddings, mtmd_bitmap_free, + mtmd_context_params_default, mtmd_decode_use_mrope, mtmd_default_marker, mtmd_free, + mtmd_gen_audio_get_info, mtmd_helper_bitmap_init_from_buf, mtmd_helper_eval_chunk_single, + mtmd_helper_eval_chunks, mtmd_helper_gen_audio_free, mtmd_helper_gen_audio_get_output, + mtmd_helper_gen_audio_init, mtmd_helper_gen_audio_reset, mtmd_helper_gen_audio_set_input, + mtmd_helper_gen_audio_step_gen, mtmd_helper_gen_audio_step_prompt, mtmd_helper_get_n_pos, + mtmd_helper_get_n_tokens, mtmd_helper_image_get_decoder_pos, mtmd_helper_init_opt_default, + mtmd_helper_log_set, mtmd_helper_video_free, mtmd_init_from_file, + mtmd_input_chunk_get_n_tokens, mtmd_input_chunk_get_tokens_image, + mtmd_input_chunk_get_tokens_text, mtmd_input_chunk_get_type, mtmd_input_chunks_free, + mtmd_input_chunks_get, mtmd_input_chunks_init, mtmd_input_chunks_size, mtmd_tokenize, + skippy_abi_features, skippy_apply_chat_template_json, skippy_backend_device_at, skippy_backend_device_count, skippy_decode_batch_sampled, skippy_decode_step_frame_batch_sampled, skippy_decode_step_frame_sampled, skippy_decode_step_frame_sampled_mtp, skippy_decode_step_sampled, @@ -152,25 +163,26 @@ pub use static_bindings::{ skippy_model_info_tensor_count, skippy_model_input_activation_boundary, skippy_model_llama_model, skippy_model_open, skippy_model_open_from_parts, skippy_model_open_from_source, skippy_model_output_activation_boundary, - skippy_ngram_cache_append, skippy_ngram_cache_create, skippy_ngram_cache_draft, - skippy_ngram_cache_free, skippy_ngram_cache_reset, skippy_parse_chat_response_json, - skippy_prefill_chunk, skippy_prefill_chunk_frame, skippy_prefill_chunk_frame_sampled, - skippy_prefill_chunk_frame_sampled_with_positions, skippy_prefill_chunk_frame_with_positions, - skippy_retire_verify_checkpoint, skippy_session_batch_size, - skippy_session_begin_external_decode, skippy_session_configure_chat_sampling, - skippy_session_copy_output_activation_frame, skippy_session_create, - skippy_session_create_from_resident_prefix, skippy_session_drop_sequence, + skippy_model_workload_info_v1, skippy_ngram_cache_append, skippy_ngram_cache_create, + skippy_ngram_cache_draft, skippy_ngram_cache_free, skippy_ngram_cache_reset, + skippy_parse_chat_response_json, skippy_prefill_chunk, skippy_prefill_chunk_frame, + skippy_prefill_chunk_frame_sampled, skippy_prefill_chunk_frame_sampled_with_positions, + skippy_prefill_chunk_frame_with_positions, skippy_retire_verify_checkpoint, + skippy_session_batch_size, skippy_session_begin_external_decode, + skippy_session_configure_chat_sampling, skippy_session_copy_output_activation_frame, + skippy_session_create, skippy_session_create_from_resident_prefix, + skippy_session_drop_sequence, skippy_session_embed, skippy_session_encode_prompt, skippy_session_end_external_decode, skippy_session_free, skippy_session_last_token_signal, skippy_session_llama_context, skippy_session_memory_used_cells, skippy_session_position, - skippy_session_reset, skippy_session_restore_prefix, skippy_session_sample_current, - skippy_session_save_prefix, skippy_session_sequence_id, skippy_session_set_position, - skippy_session_signal_window, skippy_slice_plan_add_layer_range, skippy_slice_plan_create, - skippy_slice_plan_free, skippy_stage_plan_describe_v1, skippy_stage_plan_free, - skippy_stage_plan_profile_at_v1, skippy_stage_plan_resident_tensor_at_v1, - skippy_stage_plan_state_at_v1, skippy_stage_plan_string_v1, - skippy_stage_plan_validate_chain_v1, skippy_stage_plan_value_at_v1, - skippy_stage_planner_create_v1, skippy_stage_planner_free, skippy_stage_planner_realize_v1, - skippy_token_is_eog, skippy_tokenize, skippy_trim_session, skippy_verify_tokens, - skippy_verify_tokens_frame_sampled, skippy_write_gguf_from_parts, + skippy_session_rerank, skippy_session_reset, skippy_session_restore_prefix, + skippy_session_sample_current, skippy_session_save_prefix, skippy_session_sequence_id, + skippy_session_set_position, skippy_session_signal_window, skippy_slice_plan_add_layer_range, + skippy_slice_plan_create, skippy_slice_plan_free, skippy_stage_plan_describe_v1, + skippy_stage_plan_free, skippy_stage_plan_profile_at_v1, + skippy_stage_plan_resident_tensor_at_v1, skippy_stage_plan_state_at_v1, + skippy_stage_plan_string_v1, skippy_stage_plan_validate_chain_v1, + skippy_stage_plan_value_at_v1, skippy_stage_planner_create_v1, skippy_stage_planner_free, + skippy_stage_planner_realize_v1, skippy_token_is_eog, skippy_tokenize, skippy_trim_session, + skippy_verify_tokens, skippy_verify_tokens_frame_sampled, skippy_write_gguf_from_parts, skippy_write_gguf_metadata_from_parts, skippy_write_slice_gguf, }; diff --git a/crates/skippy-ffi/src/multimodal.rs b/crates/skippy-ffi/src/multimodal.rs index 2651dc46c1..0a3f615b1f 100644 --- a/crates/skippy-ffi/src/multimodal.rs +++ b/crates/skippy-ffi/src/multimodal.rs @@ -22,6 +22,48 @@ pub struct MtmdHelperVideo { _private: [u8; 0], } +#[repr(C)] +pub struct MtmdHelperGenAudio { + _private: [u8; 0], +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(i32)] +pub enum MtmdGenAudioType { + None = 0, + Qwen3Tts = 1, + PocketTts = 2, +} + +#[repr(C)] +#[derive(Debug, Clone, Copy)] +pub struct MtmdGenAudioInfo { + pub audio_type: MtmdGenAudioType, + pub sample_rate: i32, + pub model_variant: *const c_char, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(i32)] +pub enum MtmdHelperGenAudioOutputType { + Pcm = 0, + Wav = 1, +} + +#[repr(C)] +#[derive(Debug, Clone, Copy)] +pub struct MtmdHelperGenAudioInput { + pub seq_id: i32, + pub prompt: *const c_char, + pub prompt_len: usize, + pub speaker_ref: *mut MtmdBitmap, + pub lang: *const c_char, + pub top_k: i32, + pub top_p: f32, + pub seed: u32, + pub out_type: MtmdHelperGenAudioOutputType, +} + /// Mirrors llama.cpp's `mtmd_helper_bitmap_wrapper`: the decoded bitmap plus /// an optional video context (non-null only for video inputs, which own the /// frame storage the bitmap points into). diff --git a/crates/skippy-ffi/src/static_bindings.rs b/crates/skippy-ffi/src/static_bindings.rs index a40b48ac0c..0ab57204e2 100644 --- a/crates/skippy-ffi/src/static_bindings.rs +++ b/crates/skippy-ffi/src/static_bindings.rs @@ -4,14 +4,19 @@ use crate::{ ActivationBoundaryDesc, ActivationDesc, BackendDevice, Error, GenerationSignalWindow, IterationRequest, KvPageDesc, LlamaLogCallback, LlamaModelQuantizeParams, Model, ModelInfo, ModelTensorSourceV1, MtmdBitmap, MtmdContext, MtmdContextParams, MtmdDecoderPos, - MtmdHelperBitmapWrapper, MtmdHelperInitOpt, MtmdHelperVideo, MtmdInputChunkType, - MtmdInputChunks, MtmdInputText, NativeMtpDraft, NgramCache, Opaque, RuntimeConfig, - SamplingConfig, Session, SlicePlan, StagePlan, StagePlanDescV1, StagePlanProfileDescV1, - StagePlanStateDescV1, StagePlanStringRefV1, StagePlanValueDescV1, StagePlanValueKind, - StagePlanner, StagePlannerConfigV1, Status, TensorInfo, TokenSignal, + MtmdGenAudioInfo, MtmdHelperBitmapWrapper, MtmdHelperGenAudio, MtmdHelperGenAudioInput, + MtmdHelperInitOpt, MtmdHelperVideo, MtmdInputChunkType, MtmdInputChunks, MtmdInputText, + NativeMtpDraft, NgramCache, Opaque, RuntimeConfig, SamplingConfig, Session, SlicePlan, + StagePlan, StagePlanDescV1, StagePlanProfileDescV1, StagePlanStateDescV1, StagePlanStringRefV1, + StagePlanValueDescV1, StagePlanValueKind, StagePlanner, StagePlannerConfigV1, Status, + TensorInfo, TokenSignal, WorkloadInfoV1, }; unsafe extern "C" { + pub fn llama_get_embeddings_ith(ctx: *mut Opaque, index: i32) -> *mut f32; + + pub fn llama_set_embeddings(ctx: *mut Opaque, embeddings: bool); + pub fn llama_log_set(log_callback: LlamaLogCallback, user_data: *mut c_void); pub fn ggml_log_set(log_callback: LlamaLogCallback, user_data: *mut c_void); @@ -123,6 +128,12 @@ unsafe extern "C" { out_desc: *mut ActivationBoundaryDesc, ) -> bool; + pub fn skippy_model_workload_info_v1( + model: *const Model, + out_info: *mut WorkloadInfoV1, + out_error: *mut *mut Error, + ) -> Status; + pub fn skippy_session_create( model: *mut Model, out_session: *mut *mut Session, @@ -177,6 +188,33 @@ unsafe extern "C" { out_error: *mut *mut Error, ) -> Status; + pub fn skippy_session_embed( + session: *mut Session, + token_ids: *const i32, + token_count: usize, + output: *mut f32, + output_capacity: usize, + out_dimensions: *mut usize, + out_error: *mut *mut Error, + ) -> Status; + + pub fn skippy_session_rerank( + session: *mut Session, + query: *const c_char, + document: *const c_char, + out_score: *mut f32, + out_token_count: *mut usize, + out_error: *mut *mut Error, + ) -> Status; + + pub fn skippy_session_encode_prompt( + session: *mut Session, + token_ids: *const i32, + token_count: usize, + out_decoder_start_token: *mut i32, + out_error: *mut *mut Error, + ) -> Status; + pub fn skippy_session_reset(session: *mut Session, out_error: *mut *mut Error) -> Status; pub fn skippy_session_free(session: *mut Session, out_error: *mut *mut Error) -> Status; @@ -711,6 +749,40 @@ unsafe extern "C" { pub fn mtmd_context_params_default() -> MtmdContextParams; + pub fn mtmd_gen_audio_get_info(ctx: *const MtmdContext) -> MtmdGenAudioInfo; + + pub fn mtmd_helper_gen_audio_init( + lctx: *mut Opaque, + mctx: *mut MtmdContext, + ) -> *mut MtmdHelperGenAudio; + + pub fn mtmd_helper_gen_audio_free(ctx: *mut MtmdHelperGenAudio); + + pub fn mtmd_helper_gen_audio_reset(ctx: *mut MtmdHelperGenAudio); + + pub fn mtmd_helper_gen_audio_set_input( + ctx: *mut MtmdHelperGenAudio, + input: *const MtmdHelperGenAudioInput, + ) -> i32; + + pub fn mtmd_helper_gen_audio_step_prompt(ctx: *mut MtmdHelperGenAudio, n_batch: i32) -> i32; + + pub fn mtmd_helper_gen_audio_step_gen( + ctx: *mut MtmdHelperGenAudio, + sampled: i32, + h_state_in: *const f32, + h_state_out: *mut *const f32, + out_stop: *mut bool, + ) -> i32; + + pub fn mtmd_helper_gen_audio_get_output( + ctx: *mut MtmdHelperGenAudio, + out_sample_rate: *mut i32, + out_data: *mut *const c_char, + out_data_len: *mut usize, + out_n_samples: *mut i64, + ) -> i32; + pub fn mtmd_init_from_file( mmproj_fname: *const c_char, text_model: *const Opaque, diff --git a/crates/skippy-ffi/src/tests.rs b/crates/skippy-ffi/src/tests.rs index d9e87260f6..c5b65ac762 100644 --- a/crates/skippy-ffi/src/tests.rs +++ b/crates/skippy-ffi/src/tests.rs @@ -4,7 +4,8 @@ use crate::{ ABI_VERSION_MAJOR, ABI_VERSION_MINOR, ABI_VERSION_PATCH, AbiVersion, ActivationBoundaryDesc, StagePlanDescV1, StagePlanProfileDescV1, StagePlanStateDescV1, StagePlanStateKind, StagePlanStringRefV1, StagePlanValueDescV1, StagePlannerConfigV1, StagePlannerProfileV1, - StagePlannerTensorV1, runtime_abi_supported, + StagePlannerTensorV1, WORKLOAD_INFO_V1_ABI_VERSION, WorkloadInfoV1, WorkloadKind, + WorkloadPooling, runtime_abi_supported, }; #[cfg(target_pointer_width = "64")] @@ -24,6 +25,25 @@ const fn version(major: u32, minor: u32, patch: u32) -> AbiVersion { } } +#[test] +fn workload_descriptor_matches_native_layout_and_discriminants() { + assert_eq!(WORKLOAD_INFO_V1_ABI_VERSION, 1); + assert_eq!(size_of::(), 28); + assert_eq!(offset_of!(WorkloadInfoV1, abi_version), 0); + assert_eq!(offset_of!(WorkloadInfoV1, struct_size), 4); + assert_eq!(offset_of!(WorkloadInfoV1, kind), 8); + assert_eq!(offset_of!(WorkloadInfoV1, pooling), 12); + assert_eq!(offset_of!(WorkloadInfoV1, output_dimensions), 16); + assert_eq!(offset_of!(WorkloadInfoV1, classifier_outputs), 20); + assert_eq!(offset_of!(WorkloadInfoV1, has_encoder), 24); + assert_eq!(WorkloadKind::CausalGeneration as i32, 0); + assert_eq!(WorkloadKind::Embedding as i32, 1); + assert_eq!(WorkloadKind::Rerank as i32, 2); + assert_eq!(WorkloadKind::EncoderDecoder as i32, 3); + assert_eq!(WorkloadPooling::Unspecified as i32, -1); + assert_eq!(WorkloadPooling::Rank as i32, 4); +} + #[test] fn accepts_current_patch_runtime() { assert!(runtime_abi_supported(version( diff --git a/crates/skippy-quantize/src/compose_mtp.rs b/crates/skippy-quantize/src/compose_mtp.rs index cfc637a0cb..d14ae557f9 100644 --- a/crates/skippy-quantize/src/compose_mtp.rs +++ b/crates/skippy-quantize/src/compose_mtp.rs @@ -235,44 +235,34 @@ fn per_layer_suffix(key: &str) -> Option<&'static str> { fn extend_array_kv(kv: &mut GgufKv, mtp_kv: &[GgufKv], layer_count: usize) -> Result<()> { match kv { - GgufKv::ArrayU32 { key, value } => { - if value.len() == layer_count { - let mtp_value = mtp_layer_integer(mtp_kv, key); - let fallback = u64::from(value[value.len() - 1]); - value.push( - u32::try_from(mtp_value.unwrap_or(fallback)) - .context("per-layer array entry overflows uint32")?, - ); - } + GgufKv::ArrayU32 { key, value } if value.len() == layer_count && !value.is_empty() => { + let mtp_value = mtp_layer_integer(mtp_kv, key); + let fallback = u64::from(value[value.len() - 1]); + value.push( + u32::try_from(mtp_value.unwrap_or(fallback)) + .context("per-layer array entry overflows uint32")?, + ); } - GgufKv::ArrayI32 { key, value } => { - if value.len() == layer_count { - let mtp_value = mtp_layer_integer(mtp_kv, key); - let fallback = u64::try_from(i64::from(value[value.len() - 1])) - .context("negative per-layer array entry")?; - value.push( - i32::try_from(mtp_value.unwrap_or(fallback)) - .context("per-layer array entry overflows int32")?, - ); - } + GgufKv::ArrayI32 { key, value } if value.len() == layer_count && !value.is_empty() => { + let mtp_value = mtp_layer_integer(mtp_kv, key); + let fallback = u64::try_from(i64::from(value[value.len() - 1])) + .context("negative per-layer array entry")?; + value.push( + i32::try_from(mtp_value.unwrap_or(fallback)) + .context("per-layer array entry overflows int32")?, + ); } - GgufKv::ArrayF32 { value, .. } => { - if value.len() == layer_count && !value.is_empty() { - let last = value[value.len() - 1]; - value.push(last); - } + GgufKv::ArrayF32 { value, .. } if value.len() == layer_count && !value.is_empty() => { + let last = value[value.len() - 1]; + value.push(last); } - GgufKv::ArrayBool { value, .. } => { - if value.len() == layer_count && !value.is_empty() { - let last = value[value.len() - 1]; - value.push(last); - } + GgufKv::ArrayBool { value, .. } if value.len() == layer_count && !value.is_empty() => { + let last = value[value.len() - 1]; + value.push(last); } - GgufKv::ArrayString { value, .. } => { - if value.len() == layer_count && !value.is_empty() { - let last = value[value.len() - 1].clone(); - value.push(last); - } + GgufKv::ArrayString { value, .. } if value.len() == layer_count && !value.is_empty() => { + let last = value[value.len() - 1].clone(); + value.push(last); } // Typed-array variants cover only u32/i32/f32/bool/string; other // element widths (e.g. the u16 `attention.head_count` the Nemotron diff --git a/crates/skippy-runtime/src/lib.rs b/crates/skippy-runtime/src/lib.rs index a8d2c7dc95..bd20342f2c 100644 --- a/crates/skippy-runtime/src/lib.rs +++ b/crates/skippy-runtime/src/lib.rs @@ -39,7 +39,8 @@ pub use logging::{ set_filtered_native_logs_enabled, suppress_native_logs, unregister_filtered_native_logs, write_native_log_note, }; -pub use native::{StageModel, StageModelReader}; +pub use media::{SpeechAudio, SpeechOutputFormat, SpeechSynthesisConfig}; +pub use native::{ModelWorkload, PoolingType, StageModel, StageModelReader, WorkloadInfo}; pub use native_mtp::NativeMtpDraft; pub use ngram::{Cache as NgramCache, NGRAM_CACHE_MAX_NGRAM}; pub use runtime_events::{ diff --git a/crates/skippy-runtime/src/media.rs b/crates/skippy-runtime/src/media.rs index 2ed50d30fb..d236804c22 100644 --- a/crates/skippy-runtime/src/media.rs +++ b/crates/skippy-runtime/src/media.rs @@ -13,6 +13,31 @@ use crate::{ MediaPrefillFrame, SamplingConfig, }; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SpeechOutputFormat { + Wav, + PcmS16Le, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct SpeechSynthesisConfig { + pub prompt: String, + pub language: Option, + pub top_k: i32, + pub top_p: f32, + pub seed: u32, + pub output_format: SpeechOutputFormat, + pub max_frames: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SpeechAudio { + pub bytes: Vec, + pub sample_rate: u32, + pub sample_count: u64, + pub generated_frames: usize, +} + pub(crate) struct MediaProjector { pub(crate) raw: *mut skippy_ffi::MtmdContext, marker: String, @@ -110,6 +135,185 @@ impl StageModel { self.media.is_some() } + pub fn supports_speech_synthesis(&self) -> bool { + self.media.as_ref().is_some_and(|projector| { + let info = unsafe { skippy_ffi::mtmd_gen_audio_get_info(projector.raw) }; + info.audio_type != skippy_ffi::MtmdGenAudioType::None + }) + } + + pub fn synthesize_speech( + &self, + session: &mut StageSession, + config: &SpeechSynthesisConfig, + cancellation_requested: impl Fn() -> bool, + ) -> Result { + let projector = self + .media + .as_ref() + .ok_or_else(|| anyhow!("speech synthesis requires a configured projector"))?; + if !self.supports_speech_synthesis() { + return Err(anyhow!( + "configured projector does not support speech synthesis" + )); + } + if config.prompt.is_empty() || config.max_frames == 0 { + return Err(anyhow!( + "speech prompt and max_frames must not be empty or zero" + )); + } + let prompt = CString::new(config.prompt.as_bytes()) + .context("speech prompt contains an interior NUL byte")?; + let language = config + .language + .as_deref() + .map(CString::new) + .transpose() + .context("speech language contains an interior NUL byte")?; + let lctx = unsafe { skippy_ffi::skippy_session_llama_context(session.raw) }; + if lctx.is_null() { + return Err(anyhow!("speech session did not expose a llama context")); + } + + struct AudioGenerator(*mut skippy_ffi::MtmdHelperGenAudio); + impl Drop for AudioGenerator { + fn drop(&mut self) { + if !self.0.is_null() { + unsafe { skippy_ffi::mtmd_helper_gen_audio_free(self.0) }; + } + } + } + struct ExternalDecodeGuard(*mut skippy_ffi::Session); + impl Drop for ExternalDecodeGuard { + fn drop(&mut self) { + let mut error = ptr::null_mut(); + unsafe { + let _ = skippy_ffi::skippy_session_end_external_decode(self.0, &mut error); + } + free_error(error); + } + } + + session.reset()?; + unsafe { skippy_ffi::llama_set_embeddings(lctx, true) }; + let mut guard_error = ptr::null_mut(); + let status = unsafe { + skippy_ffi::skippy_session_begin_external_decode(session.raw, &mut guard_error) + }; + ensure_ok(status, guard_error)?; + let _external_decode = ExternalDecodeGuard(session.raw); + let generator = + AudioGenerator(unsafe { skippy_ffi::mtmd_helper_gen_audio_init(lctx, projector.raw) }); + if generator.0.is_null() { + return Err(anyhow!("failed to initialize speech synthesis pipeline")); + } + let output_type = match config.output_format { + SpeechOutputFormat::Wav => skippy_ffi::MtmdHelperGenAudioOutputType::Wav, + SpeechOutputFormat::PcmS16Le => skippy_ffi::MtmdHelperGenAudioOutputType::Pcm, + }; + let input = skippy_ffi::MtmdHelperGenAudioInput { + seq_id: session.native_sequence_id()?, + prompt: prompt.as_ptr(), + prompt_len: config.prompt.len(), + speaker_ref: ptr::null_mut(), + lang: language + .as_ref() + .map_or(ptr::null(), |value| value.as_ptr()), + top_k: config.top_k, + top_p: config.top_p, + seed: config.seed, + out_type: output_type, + }; + if unsafe { skippy_ffi::mtmd_helper_gen_audio_set_input(generator.0, &input) } != 0 { + return Err(anyhow!("speech synthesis rejected the input")); + } + let batch_size = + i32::try_from(session.batch_size()?).context("speech batch size exceeds i32")?; + loop { + if cancellation_requested() { + return Err(anyhow!("speech synthesis cancelled")); + } + let remaining = + unsafe { skippy_ffi::mtmd_helper_gen_audio_step_prompt(generator.0, batch_size) }; + if remaining < 0 { + return Err(anyhow!("speech prompt evaluation failed")); + } + if remaining == 0 { + break; + } + } + + let sampling = SamplingConfig { + enabled: true, + seed: config.seed, + top_k: config.top_k, + top_p: config.top_p, + ..SamplingConfig::default() + }; + let mut sampled = session.sample_current(Some(&sampling))?; + let mut hidden_state = + unsafe { skippy_ffi::llama_get_embeddings_ith(lctx, -1) }.cast_const(); + if hidden_state.is_null() { + return Err(anyhow!("speech backbone did not produce a hidden state")); + } + let mut generated_frames = 0usize; + while generated_frames < config.max_frames { + if cancellation_requested() { + return Err(anyhow!("speech synthesis cancelled")); + } + let mut next_hidden_state = ptr::null(); + let mut stop = false; + let step = unsafe { + skippy_ffi::mtmd_helper_gen_audio_step_gen( + generator.0, + sampled, + hidden_state, + &mut next_hidden_state, + &mut stop, + ) + }; + if step != 0 { + return Err(anyhow!( + "speech synthesis failed at frame {generated_frames}" + )); + } + if stop || next_hidden_state.is_null() { + break; + } + generated_frames += 1; + hidden_state = next_hidden_state; + sampled = session.sample_current(Some(&sampling))?; + } + + let mut sample_rate = 0_i32; + let mut data = ptr::null(); + let mut data_len = 0usize; + let mut sample_count = 0_i64; + let output_status = unsafe { + skippy_ffi::mtmd_helper_gen_audio_get_output( + generator.0, + &mut sample_rate, + &mut data, + &mut data_len, + &mut sample_count, + ) + }; + if output_status != 0 || data.is_null() || data_len == 0 { + return Err(anyhow!("speech synthesis produced no audio")); + } + let native_bytes = unsafe { std::slice::from_raw_parts(data.cast::(), data_len) }; + let bytes = match config.output_format { + SpeechOutputFormat::Wav => native_bytes.to_vec(), + SpeechOutputFormat::PcmS16Le => pcm_f32_to_s16le(native_bytes)?, + }; + Ok(SpeechAudio { + bytes, + sample_rate: u32::try_from(sample_rate).context("invalid speech sample rate")?, + sample_count: u64::try_from(sample_count).context("invalid speech sample count")?, + generated_frames, + }) + } + fn eval_media( &self, session: &mut StageSession, @@ -641,3 +845,46 @@ impl StageModel { }) } } + +fn pcm_f32_to_s16le(bytes: &[u8]) -> Result> { + if !bytes.len().is_multiple_of(std::mem::size_of::()) { + return Err(anyhow!("native PCM payload is not aligned to f32 samples")); + } + let mut output = Vec::with_capacity(bytes.len() / 2); + for sample in bytes.chunks_exact(4) { + let sample = f32::from_ne_bytes(sample.try_into().expect("four-byte PCM sample")); + let quantized = (sample.clamp(-1.0, 1.0) * f32::from(i16::MAX)).round() as i16; + output.extend_from_slice(&quantized.to_le_bytes()); + } + Ok(output) +} + +#[cfg(test)] +mod tests { + use super::pcm_f32_to_s16le; + + #[test] + fn pcm_conversion_clamps_and_quantizes_native_float_samples() { + let samples = [-2.0_f32, -1.0, -0.5, 0.0, 0.5, 1.0, 2.0]; + let bytes = samples + .iter() + .flat_map(|sample| sample.to_ne_bytes()) + .collect::>(); + + let converted = pcm_f32_to_s16le(&bytes).expect("aligned native PCM"); + let actual = converted + .chunks_exact(2) + .map(|sample| i16::from_le_bytes(sample.try_into().unwrap())) + .collect::>(); + + assert_eq!( + actual, + vec![-32_767, -32_767, -16_384, 0, 16_384, 32_767, 32_767] + ); + } + + #[test] + fn pcm_conversion_rejects_misaligned_native_payload() { + assert!(pcm_f32_to_s16le(&[0, 1, 2]).is_err()); + } +} diff --git a/crates/skippy-runtime/src/native.rs b/crates/skippy-runtime/src/native.rs index d8692e6943..7ca662f4ae 100644 --- a/crates/skippy-runtime/src/native.rs +++ b/crates/skippy-runtime/src/native.rs @@ -18,6 +18,72 @@ use crate::{ RuntimeEvent, Status, }; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ModelWorkload { + CausalGeneration, + Embedding, + Rerank, + EncoderDecoder, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PoolingType { + Unspecified, + None, + Mean, + Cls, + Last, + Rank, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct WorkloadInfo { + pub kind: ModelWorkload, + pub pooling: PoolingType, + pub output_dimensions: u32, + pub classifier_outputs: u32, + pub has_encoder: bool, + pub has_decoder: bool, + pub full_model_only: bool, +} + +impl TryFrom for WorkloadInfo { + type Error = anyhow::Error; + + fn try_from(raw: skippy_ffi::WorkloadInfoV1) -> Result { + if raw.abi_version != skippy_ffi::WORKLOAD_INFO_V1_ABI_VERSION + || raw.struct_size != std::mem::size_of::() as u32 + { + return Err(anyhow!( + "native workload descriptor uses an incompatible ABI" + )); + } + let kind = match raw.kind { + skippy_ffi::WorkloadKind::CausalGeneration => ModelWorkload::CausalGeneration, + skippy_ffi::WorkloadKind::Embedding => ModelWorkload::Embedding, + skippy_ffi::WorkloadKind::Rerank => ModelWorkload::Rerank, + skippy_ffi::WorkloadKind::EncoderDecoder => ModelWorkload::EncoderDecoder, + }; + let pooling = match raw.pooling { + skippy_ffi::WorkloadPooling::Unspecified => PoolingType::Unspecified, + skippy_ffi::WorkloadPooling::None => PoolingType::None, + skippy_ffi::WorkloadPooling::Mean => PoolingType::Mean, + skippy_ffi::WorkloadPooling::Cls => PoolingType::Cls, + skippy_ffi::WorkloadPooling::Last => PoolingType::Last, + skippy_ffi::WorkloadPooling::Rank => PoolingType::Rank, + }; + Ok(Self { + kind, + pooling, + output_dimensions: raw.output_dimensions, + classifier_outputs: raw.classifier_outputs, + has_encoder: raw.has_encoder, + has_decoder: raw.has_decoder, + full_model_only: raw.full_model_only, + }) + } +} + pub struct StageModel { inner: Arc, pub(crate) media: Option, @@ -106,6 +172,16 @@ impl StageModel { present.then(|| raw.into()) } + pub fn workload_info(&self) -> Result { + let mut raw = skippy_ffi::WorkloadInfoV1::default(); + let mut error = ptr::null_mut(); + let status = unsafe { + skippy_ffi::skippy_model_workload_info_v1(self.inner.raw, &mut raw, &mut error) + }; + ensure_ok(status, error)?; + raw.try_into() + } + fn from_opened_raw( raw: *mut RawModel, config: &RuntimeConfig, @@ -960,11 +1036,66 @@ impl Drop for StageModel { #[cfg(test)] mod output_capacity_tests { use super::{ - ModelStateKind, OPTIMISTIC_OUTPUT_HEADROOM, capability_from_state_probes, - classify_model_state, optimistic_chat_metadata_capacity, optimistic_chat_parse_capacity, - optimistic_chat_prompt_capacity, optimistic_token_capacity, + ModelStateKind, ModelWorkload, OPTIMISTIC_OUTPUT_HEADROOM, PoolingType, WorkloadInfo, + capability_from_state_probes, classify_model_state, optimistic_chat_metadata_capacity, + optimistic_chat_parse_capacity, optimistic_chat_prompt_capacity, optimistic_token_capacity, }; + #[test] + fn workload_descriptor_converts_all_native_classes_and_pooling_modes() { + let cases = [ + ( + skippy_ffi::WorkloadKind::CausalGeneration, + ModelWorkload::CausalGeneration, + ), + ( + skippy_ffi::WorkloadKind::Embedding, + ModelWorkload::Embedding, + ), + (skippy_ffi::WorkloadKind::Rerank, ModelWorkload::Rerank), + ( + skippy_ffi::WorkloadKind::EncoderDecoder, + ModelWorkload::EncoderDecoder, + ), + ]; + for (raw_kind, expected_kind) in cases { + let converted = WorkloadInfo::try_from(skippy_ffi::WorkloadInfoV1 { + kind: raw_kind, + pooling: skippy_ffi::WorkloadPooling::Mean, + output_dimensions: 768, + classifier_outputs: 2, + has_encoder: true, + has_decoder: false, + full_model_only: true, + ..Default::default() + }) + .expect("valid workload descriptor"); + + assert_eq!(converted.kind, expected_kind); + assert_eq!(converted.pooling, PoolingType::Mean); + assert_eq!(converted.output_dimensions, 768); + assert_eq!(converted.classifier_outputs, 2); + assert!(converted.has_encoder); + assert!(!converted.has_decoder); + assert!(converted.full_model_only); + } + } + + #[test] + fn workload_descriptor_rejects_incompatible_layout_versions() { + let invalid_version = skippy_ffi::WorkloadInfoV1 { + abi_version: skippy_ffi::WORKLOAD_INFO_V1_ABI_VERSION + 1, + ..Default::default() + }; + assert!(WorkloadInfo::try_from(invalid_version).is_err()); + + let invalid_size = skippy_ffi::WorkloadInfoV1 { + struct_size: 0, + ..Default::default() + }; + assert!(WorkloadInfo::try_from(invalid_size).is_err()); + } + #[test] fn loaded_model_flags_classify_state_without_family_names() { assert_eq!( diff --git a/crates/skippy-runtime/src/session.rs b/crates/skippy-runtime/src/session.rs index 56d40b7071..4236bf36f5 100644 --- a/crates/skippy-runtime/src/session.rs +++ b/crates/skippy-runtime/src/session.rs @@ -87,6 +87,73 @@ impl StageSession { Ok(()) } + pub fn embed(&mut self, token_ids: &[i32], dimensions: usize) -> Result> { + if dimensions == 0 { + return Err(anyhow!("embedding dimensions must be greater than zero")); + } + let mut output = vec![0.0_f32; dimensions]; + let mut actual_dimensions = 0usize; + let mut error = ptr::null_mut(); + let status = unsafe { + skippy_ffi::skippy_session_embed( + self.raw, + token_ids.as_ptr(), + token_ids.len(), + output.as_mut_ptr(), + output.len(), + &mut actual_dimensions, + &mut error, + ) + }; + ensure_ok(status, error)?; + if actual_dimensions != dimensions { + return Err(anyhow!( + "native embedding dimensions changed from {dimensions} to {actual_dimensions}" + )); + } + self.token_count = u64::try_from(token_ids.len()).context("token count exceeds u64")?; + Ok(output) + } + + pub fn rerank(&mut self, query: &str, document: &str) -> Result<(f32, usize)> { + let query = CString::new(query).context("rerank query contains an interior NUL byte")?; + let document = + CString::new(document).context("rerank document contains an interior NUL byte")?; + let mut score = 0.0_f32; + let mut token_count = 0usize; + let mut error = ptr::null_mut(); + let status = unsafe { + skippy_ffi::skippy_session_rerank( + self.raw, + query.as_ptr(), + document.as_ptr(), + &mut score, + &mut token_count, + &mut error, + ) + }; + ensure_ok(status, error)?; + self.token_count = u64::try_from(token_count).context("token count exceeds u64")?; + Ok((score, token_count)) + } + + pub fn encode_prompt(&mut self, token_ids: &[i32]) -> Result { + let mut decoder_start_token = 0_i32; + let mut error = ptr::null_mut(); + let status = unsafe { + skippy_ffi::skippy_session_encode_prompt( + self.raw, + token_ids.as_ptr(), + token_ids.len(), + &mut decoder_start_token, + &mut error, + ) + }; + ensure_ok(status, error)?; + self.token_count = 0; + Ok(decoder_start_token) + } + pub fn configure_chat_sampling( &mut self, metadata_json: &str, diff --git a/crates/skippy-server/README.md b/crates/skippy-server/README.md index 76e5972eb5..c103a6fb46 100644 --- a/crates/skippy-server/README.md +++ b/crates/skippy-server/README.md @@ -142,8 +142,9 @@ deadline handling. macOS interface-scoped socket option. In a multi-NIC lab, set `bind_addr` to the private LAN address, such as `192.168.0.x:19031`, so both inbound serving and outbound stage-to-stage traffic are pinned to that interface. -- `serve-openai` exposes `/v1/models`, `/v1/chat/completions`, and - `/v1/completions` using the shared `openai-frontend` crate for a local +- `serve-openai` exposes model discovery, chat/completions, Responses, + embeddings, rerank, and audio endpoints using the shared `openai-frontend` + crate for a local final/single-stage config with no downstream peer. Split serving uses embedded stage-0 OpenAI serving from `serve-binary --openai-bind-addr` because generation-7 prediction returns flow directly from the final stage to stage 0. diff --git a/crates/skippy-server/src/embedded.rs b/crates/skippy-server/src/embedded.rs index afa8c4efe2..2eac46ffbe 100644 --- a/crates/skippy-server/src/embedded.rs +++ b/crates/skippy-server/src/embedded.rs @@ -8,7 +8,7 @@ use anyhow::{Context, Result}; use axum::Router; use openai_frontend::{OpenAiBackend, OpenAiFrontendConfig, OpenAiLifecycleObserver}; use skippy_protocol::{StageConfig, StageTopology}; -use skippy_runtime::{ActivationBoundaryDesc, MtpSource}; +use skippy_runtime::{ActivationBoundaryDesc, MtpSource, WorkloadInfo}; use tokio::{sync::oneshot, task::JoinHandle}; use crate::{ @@ -131,6 +131,23 @@ impl SkippyRuntimeHandle { .output_activation_boundary() } + /// Returns the runtime-probed workload contract for the loaded model. + pub fn workload_info(&self) -> Result { + self.runtime + .lock() + .expect("runtime lock poisoned") + .workload_info() + } + + /// True only when the loaded multimodal projector exposes llama.cpp's + /// audio-generation helper contract. + pub fn supports_speech_synthesis(&self) -> bool { + self.runtime + .lock() + .expect("runtime lock poisoned") + .supports_speech_synthesis() + } + /// Assemble a ready handle around an already-loaded runtime. /// /// Shared by both loaders so the stats cache is primed exactly once, in one diff --git a/crates/skippy-server/src/frontend/backend.rs b/crates/skippy-server/src/frontend/backend.rs index b3d42aa26b..4896b071b0 100644 --- a/crates/skippy-server/src/frontend/backend.rs +++ b/crates/skippy-server/src/frontend/backend.rs @@ -16,6 +16,7 @@ use crate::frontend::generation::GenerationSessionLockEntry; use crate::frontend::generation::GenerationStream; use crate::frontend::generation::GenerationStreamEvent; use crate::frontend::generation::GenerationTokenLimit; +use crate::frontend::generation::OpenAiBackendMode; use crate::frontend::generation::OpenAiCacheHints; use crate::frontend::generation::OpenAiGenerationIds; use crate::frontend::generation::PhaseTimer; @@ -35,7 +36,7 @@ use crate::frontend::generation::template_exposes_reasoning; use crate::frontend::request::{ apply_chat_request_defaults, apply_completion_request_defaults, chat_sampling_config, chat_template_options, completion_sampling_config, ensure_chat_runtime_features_supported, - ensure_completion_runtime_features_supported, + ensure_completion_runtime_features_supported, sampling_config, }; use crate::runtime_state::RuntimeSessionStats; use crate::telemetry::Telemetry; @@ -44,6 +45,11 @@ use crate::telemetry::now_unix_nanos; use async_trait::async_trait; use futures_util::StreamExt; use futures_util::stream; +use openai_frontend::AudioFormat; +use openai_frontend::AudioResponse; +use openai_frontend::AudioSpeechRequest; +use openai_frontend::AudioTranscriptionRequest; +use openai_frontend::AudioTranscriptionResponse; use openai_frontend::ChatCompletionOutcome; use openai_frontend::ChatCompletionRequest; use openai_frontend::ChatCompletionResponse; @@ -52,11 +58,18 @@ use openai_frontend::ChatExchangeRoute; use openai_frontend::CompletionRequest; use openai_frontend::CompletionResponse; use openai_frontend::CompletionStream; +use openai_frontend::Embedding; +use openai_frontend::EmbeddingInput; +use openai_frontend::EmbeddingResponse; +use openai_frontend::EmbeddingsRequest; use openai_frontend::ModelObject; use openai_frontend::OpenAiBackend; use openai_frontend::OpenAiError; use openai_frontend::OpenAiRequestContext; use openai_frontend::OpenAiResult; +use openai_frontend::RerankRequest; +use openai_frontend::RerankResponse; +use openai_frontend::RerankResult; use openai_frontend::TerminalGuard; use openai_frontend::TerminalGuardedChatStream; use openai_frontend::apply_chat_hook_outcome; @@ -65,7 +78,10 @@ use openai_frontend::chat_mesh_hooks_enabled; use serde_json::Value; use serde_json::json; use skippy_metrics::attr as attr_key; -use skippy_runtime::SamplingConfig; +use skippy_runtime::{ + MediaInput, ModelWorkload, SamplingConfig, SpeechOutputFormat, SpeechSynthesisConfig, + WorkloadInfo, +}; use std::collections::BTreeMap; use std::sync::Arc; use std::sync::Mutex; @@ -81,6 +97,8 @@ use tokio::sync::mpsc; use tokio::sync::mpsc::error::TrySendError; use tokio::task; +mod non_chat; + fn request_cancelled_error() -> OpenAiError { OpenAiError::cancelled("request cancelled") } @@ -1108,6 +1126,211 @@ impl OpenAiBackend for StageOpenAiBackend { generation_event_to_completion_chunk(event, &model) }))) } + + async fn embeddings( + &self, + request: EmbeddingsRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.ensure_model(&request.model)?; + let info = self.ensure_local_workload(ModelWorkload::Embedding)?; + let expected_dimensions = usize::try_from(info.output_dimensions) + .map_err(|_| OpenAiError::backend("embedding dimensions exceed usize"))?; + if request + .dimensions + .is_some_and(|requested| requested != expected_dimensions) + { + return Err(OpenAiError::unsupported(format!( + "model exposes {expected_dimensions} embedding dimensions; dimensionality reduction is not supported" + ))); + } + + let model = request.model.clone(); + let encoding_format = request.encoding_format.clone(); + let backend = self.clone(); + let token_inputs = task::spawn_blocking(move || backend.prepare_embedding_inputs(request)) + .await + .map_err(|error| { + OpenAiError::backend(format!("embedding tokenization task failed: {error}")) + })??; + let prompt_tokens = token_inputs.iter().map(Vec::len).sum::(); + let max_input_tokens = token_inputs.iter().map(Vec::len).max().unwrap_or_default(); + let ids = generation_ids(OpenAiCacheHints::default(), None, &context); + let cancellation = context.cancellation_token(); + let embeddings = self + .run_local_workload( + context, + ids, + max_input_tokens, + move |runtime, session_id| { + non_chat::collect_workload_batch( + token_inputs.iter().enumerate(), + &cancellation, + |(index, tokens)| { + runtime + .embed(session_id, tokens, expected_dimensions) + .map(|values| Embedding { values, index }) + }, + ) + }, + ) + .await?; + Ok(EmbeddingResponse::from_embeddings( + model, + embeddings, + u32::try_from(prompt_tokens).unwrap_or(u32::MAX), + &encoding_format, + )) + } + + async fn rerank( + &self, + request: RerankRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.ensure_model(&request.model)?; + self.ensure_local_workload(ModelWorkload::Rerank)?; + let prompt_tokens_estimate = request + .documents + .iter() + .filter_map(|document| document.text().ok()) + .map(|document| { + request + .query + .len() + .saturating_add(document.len()) + .div_ceil(3) + }) + .max() + .unwrap_or(1); + let ids = generation_ids(OpenAiCacheHints::default(), None, &context); + let cancellation = context.cancellation_token(); + let query = request.query.clone(); + let documents = request.documents.clone(); + let mut scored = self + .run_local_workload( + context, + ids, + prompt_tokens_estimate, + move |runtime, session_id| { + non_chat::collect_workload_batch( + documents.iter().enumerate(), + &cancellation, + |(index, document)| { + let (relevance_score, token_count) = + runtime.rerank(session_id, &query, document.text()?)?; + Ok((index, relevance_score, token_count)) + }, + ) + }, + ) + .await?; + let prompt_tokens = scored.iter().map(|(_, _, count)| count).sum::(); + scored.sort_by(|left, right| right.1.total_cmp(&left.1).then(left.0.cmp(&right.0))); + scored.truncate(request.top_n.unwrap_or(scored.len()).min(scored.len())); + let results = scored + .into_iter() + .map(|(index, relevance_score, _)| RerankResult { + index, + relevance_score, + document: request + .return_documents + .then(|| request.documents[index].clone()), + }) + .collect(); + Ok(RerankResponse { + id: format!("rerank-{}", uuid::Uuid::new_v4().simple()), + results, + usage: openai_frontend::Usage { + prompt_tokens: u32::try_from(prompt_tokens).unwrap_or(u32::MAX), + completion_tokens: 0, + total_tokens: u32::try_from(prompt_tokens).unwrap_or(u32::MAX), + ..openai_frontend::Usage::default() + }, + }) + } + + async fn audio_speech( + &self, + request: AudioSpeechRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.ensure_model(&request.model)?; + if !self.has_unsplit_full_model_topology() { + return Err(OpenAiError::unsupported( + "speech synthesis currently requires an unsplit local runtime", + )); + } + if (request.speed - 1.0).abs() > f32::EPSILON { + return Err(OpenAiError::unsupported( + "speech speed control is not supported by the native model", + )); + } + non_chat::validate_speech_voice(&request.voice)?; + let output_format = match request.response_format { + AudioFormat::Wav => SpeechOutputFormat::Wav, + AudioFormat::Pcm => SpeechOutputFormat::PcmS16Le, + _ => { + return Err(OpenAiError::unsupported( + "native speech synthesis currently supports wav and pcm output", + )); + } + }; + { + let runtime = self + .runtime + .lock() + .map_err(|_| OpenAiError::backend("runtime lock poisoned"))?; + if runtime.input_activation_boundary().is_some() + || runtime.output_activation_boundary().is_some() + || !runtime.supports_speech_synthesis() + { + return Err(OpenAiError::unsupported( + "model does not expose full-model speech synthesis", + )); + } + } + let content_type = request.response_format.content_type().to_string(); + let prompt_tokens_estimate = request.input.len().div_ceil(3).max(1); + let ids = generation_ids(OpenAiCacheHints::default(), None, &context); + let cancellation = context.clone(); + let config = SpeechSynthesisConfig { + prompt: request.input, + language: None, + top_k: 20, + top_p: 0.8, + seed: u32::MAX, + output_format, + max_frames: 512, + }; + let audio = self + .run_local_workload( + context, + ids, + prompt_tokens_estimate, + move |runtime, session_id| { + runtime.synthesize_speech(session_id, &config, || cancellation.is_cancelled()) + }, + ) + .await?; + AudioResponse::new(audio.bytes, content_type) + } + + async fn audio_transcription( + &self, + request: AudioTranscriptionRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.audio_to_text(request, false, context).await + } + + async fn audio_translation( + &self, + request: AudioTranscriptionRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.audio_to_text(request, true, context).await + } } impl StageOpenAiBackend { diff --git a/crates/skippy-server/src/frontend/backend/non_chat.rs b/crates/skippy-server/src/frontend/backend/non_chat.rs new file mode 100644 index 0000000000..cc712a551d --- /dev/null +++ b/crates/skippy-server/src/frontend/backend/non_chat.rs @@ -0,0 +1,436 @@ +use super::*; + +pub(super) fn collect_workload_batch( + items: I, + cancellation: &openai_frontend::CancellationToken, + mut run_item: F, +) -> anyhow::Result> +where + I: IntoIterator, + F: FnMut(I::Item) -> anyhow::Result, +{ + let mut results = Vec::new(); + for item in items { + // Native calls cannot be interrupted mid-item; stop before the next + // one so a disconnected client does not hold the runtime for a batch. + if cancellation.is_cancelled() { + return Err(request_cancelled_error().into()); + } + results.push(run_item(item)?); + } + Ok(results) +} + +fn workload_error(error: anyhow::Error) -> OpenAiError { + if let Some(openai_error) = error.downcast_ref::() { + return openai_error.clone(); + } + OpenAiError::backend(format!("workload execution failed: {error:#}")) +} + +pub(super) fn validate_speech_voice(voice: &str) -> OpenAiResult<()> { + if voice == "default" { + Ok(()) + } else { + Err(OpenAiError::unsupported( + "native speech synthesis currently supports only voice 'default'; speaker selection is not implemented", + ) + .with_param("voice")) + } +} + +impl StageOpenAiBackend { + pub(in crate::frontend) fn has_unsplit_full_model_topology(&self) -> bool { + fn is_unsplit(config: &skippy_protocol::StageConfig) -> bool { + config.stage_index == 0 + && config.layer_start == 0 + && config.layer_end > 0 + && !config.filter_tensors_on_load + && config.upstream.is_none() + && config.downstream.is_none() + } + + // The mesh serves a complete local GGUF through EmbeddedStageZero as + // well as through LocalRuntime. The mode name alone cannot tell us + // whether model execution is distributed. + is_unsplit(&self.config) + && match &self.mode { + OpenAiBackendMode::LocalRuntime => true, + OpenAiBackendMode::EmbeddedStageZero { config, .. } => is_unsplit(config), + } + } + + pub(super) async fn audio_to_text( + &self, + request: AudioTranscriptionRequest, + translate_to_english: bool, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.ensure_model(&request.model)?; + if !self.has_unsplit_full_model_topology() { + return Err(OpenAiError::unsupported( + "audio transcription currently requires an unsplit local runtime", + )); + } + { + let runtime = self + .runtime + .lock() + .map_err(|_| OpenAiError::backend("runtime lock poisoned"))?; + if runtime.input_activation_boundary().is_some() + || runtime.output_activation_boundary().is_some() + || !runtime.has_media_projector() + { + return Err(OpenAiError::unsupported( + "model does not expose full-model multimodal audio input", + )); + } + } + + let instruction = audio_text_instruction(&request, translate_to_english); + let mut chat_request = ChatCompletionRequest { + model: request.model.clone(), + messages: vec![audio_text_user_message(instruction)], + temperature: request.temperature, + ..Default::default() + }; + apply_chat_request_defaults(&mut chat_request, &self.request_defaults)?; + let template_options = chat_template_options(&chat_request, &self.request_defaults)?; + let mut prompt = self + .prepare_chat_prompt_offloaded(&chat_request, template_options) + .await?; + prompt.media = vec![MediaInput { + bytes: request.file, + }]; + let sampling = sampling_config( + chat_request.temperature, + chat_request.top_p, + chat_request.presence_penalty, + chat_request.frequency_penalty, + chat_request.seed, + chat_request.logit_bias.as_ref(), + &chat_request.extra, + )?; + let ids = generation_ids(OpenAiCacheHints::default(), None, &context); + let output = self + .run_generation( + prompt, + GenerationTokenLimit::from_request(None, self.default_max_tokens), + None, + sampling, + None, + context, + ids, + ) + .await?; + Ok(AudioTranscriptionResponse { + text: audio_transcript_text(&output.text), + }) + } + + pub(in crate::frontend) fn ensure_local_workload( + &self, + expected: ModelWorkload, + ) -> OpenAiResult { + if !self.has_unsplit_full_model_topology() { + return Err(OpenAiError::unsupported( + "non-chat workloads currently require an unsplit local runtime", + )); + } + let runtime = self + .runtime + .lock() + .map_err(|_| OpenAiError::backend("runtime lock poisoned"))?; + if runtime.input_activation_boundary().is_some() + || runtime.output_activation_boundary().is_some() + { + return Err(OpenAiError::unsupported( + "non-chat workloads currently require an unsplit full model", + )); + } + let info = runtime + .workload_info() + .map_err(|error| OpenAiError::backend(format!("read model workload: {error:#}")))?; + if info.kind != expected { + return Err(OpenAiError::unsupported(format!( + "model workload is {:?}; endpoint requires {:?}", + info.kind, expected + ))); + } + Ok(info) + } + + pub(super) fn prepare_embedding_inputs( + &self, + request: EmbeddingsRequest, + ) -> OpenAiResult>> { + let reader = self + .runtime + .lock() + .map_err(|_| OpenAiError::backend("runtime lock poisoned"))? + .model + .reader(); + match request.input { + EmbeddingInput::Text(text) => reader + .tokenize(&text, true) + .map(|tokens| vec![tokens]) + .map_err(|error| OpenAiError::backend(format!("tokenize embedding: {error:#}"))), + EmbeddingInput::Texts(texts) => texts + .into_iter() + .map(|text| { + reader.tokenize(&text, true).map_err(|error| { + OpenAiError::backend(format!("tokenize embedding: {error:#}")) + }) + }) + .collect(), + EmbeddingInput::Tokens(tokens) => Ok(vec![tokens]), + EmbeddingInput::TokenArrays(tokens) => Ok(tokens), + } + } + + pub(super) async fn run_local_workload( + &self, + context: OpenAiRequestContext, + ids: OpenAiGenerationIds, + prompt_tokens: usize, + work: F, + ) -> OpenAiResult + where + T: Send + 'static, + F: FnOnce(&mut crate::runtime_state::RuntimeState, &str) -> anyhow::Result + + Send + + 'static, + { + let cancellation = context.cancellation_token(); + let (permit, session_permit) = self + .acquire_generation_admission( + &ids, + &cancellation, + GenerationAdmissionWork::new(prompt_tokens.max(1), 0), + GenerationAdmissionScheduling::default(), + ) + .await?; + let runtime = Arc::clone(&self.runtime); + let session_id = ids.session_id_string(); + let worker_context = context.clone(); + let result = run_blocking_generation_worker(permit, context, move |token| { + let _session_permit = session_permit; + if token.is_cancelled() { + return Err(request_cancelled_error()); + } + let mut runtime = runtime + .lock() + .map_err(|_| OpenAiError::backend("runtime lock poisoned"))?; + let result = work(&mut runtime, &session_id).map_err(workload_error); + let cleanup = runtime.drop_session_timed(&session_id).map_err(|error| { + OpenAiError::backend(format!("workload session cleanup failed: {error:#}")) + }); + match (result, cleanup) { + (Ok(value), Ok(_)) => Ok(value), + (Err(error), _) => Err(error), + (Ok(_), Err(error)) => Err(error), + } + }) + .await + .map_err(|error| OpenAiError::backend(format!("workload task failed: {error}")))?; + if worker_context.is_cancelled() { + Err(request_cancelled_error()) + } else { + result + } + } +} + +fn audio_transcript_text(raw: &str) -> String { + let text = raw.trim(); + for prefix in ["The text is:", "The audio is:"] { + if let Some(quoted) = text.strip_prefix(prefix).map(str::trim) + && let Some(inner) = quoted + .strip_prefix('"') + .and_then(|value| value.strip_suffix('"')) + { + return inner.to_string(); + } + } + text.to_string() +} + +fn audio_text_instruction( + request: &AudioTranscriptionRequest, + translate_to_english: bool, +) -> String { + let mut instruction = if translate_to_english { + "Translate the supplied audio into English. Return only the English translation." + .to_string() + } else { + let language = request + .language + .as_deref() + .filter(|language| !language.trim().is_empty()) + .map_or_else(String::new, |language| format!(" (language: {language})")); + format!("Transcribe audio to text{language}") + }; + if let Some(prompt) = request + .prompt + .as_deref() + .filter(|prompt| !prompt.trim().is_empty()) + { + instruction.push_str(" Use this context when resolving names or terminology: "); + instruction.push_str(prompt); + } + instruction +} + +fn audio_text_user_message(instruction: String) -> openai_frontend::ChatMessage { + openai_frontend::ChatMessage { + role: "user".to_string(), + // The audio marker must follow the instruction. Upstream llama.cpp's + // transcription route appends the media marker to the user prompt; + // reversing these parts can make Ultravox ignore the supplied audio. + content: Some(openai_frontend::MessageContent::Parts(vec![ + openai_frontend::MessageContentPart { + content_type: "text".to_string(), + text: Some(instruction), + extra: BTreeMap::new(), + }, + openai_frontend::MessageContentPart { + content_type: "input_audio".to_string(), + text: None, + extra: BTreeMap::from([( + "input_audio".to_string(), + json!({"data": "AA==", "format": "wav"}), + )]), + }, + ])), + extra: BTreeMap::new(), + } +} + +#[cfg(test)] +mod tests { + use super::{ + audio_text_instruction, audio_text_user_message, audio_transcript_text, + collect_workload_batch, validate_speech_voice, workload_error, + }; + use openai_frontend::{AudioTranscriptionRequest, MessageContent}; + + fn audio_request() -> AudioTranscriptionRequest { + AudioTranscriptionRequest { + model: "ultravox".to_string(), + file: vec![1], + filename: Some("sample.wav".to_string()), + language: None, + prompt: None, + response_format: "json".to_string(), + temperature: Some(0.0), + } + } + + #[test] + fn transcription_uses_upstream_default_prompt_before_audio_marker() { + let instruction = audio_text_instruction(&audio_request(), false); + assert_eq!(instruction, "Transcribe audio to text"); + + let message = audio_text_user_message(instruction); + let Some(MessageContent::Parts(parts)) = message.content else { + panic!("audio request must use multipart chat content"); + }; + assert_eq!(parts.len(), 2); + assert_eq!(parts[0].content_type, "text"); + assert_eq!(parts[0].text.as_deref(), Some("Transcribe audio to text")); + assert_eq!(parts[1].content_type, "input_audio"); + assert!(parts[1].extra.contains_key("input_audio")); + } + + #[test] + fn transcription_removes_only_confirmed_quoted_ultravox_wrappers() { + assert_eq!( + audio_transcript_text("The text is: \"The mesh is ready\""), + "The mesh is ready" + ); + assert_eq!( + audio_transcript_text("The audio is: \"The mesh is ready\""), + "The mesh is ready" + ); + assert_eq!( + audio_transcript_text(" The mesh is ready "), + "The mesh is ready" + ); + assert_eq!( + audio_transcript_text("The text is: The mesh is ready"), + "The text is: The mesh is ready" + ); + assert_eq!( + audio_transcript_text("I heard: \"The mesh is ready\""), + "I heard: \"The mesh is ready\"" + ); + } + + #[test] + fn transcription_preserves_empty_text_without_an_artificial_wrapper() { + assert_eq!(audio_transcript_text(" "), ""); + assert_eq!(audio_transcript_text("The text is: \"\""), ""); + assert_eq!(audio_transcript_text("The audio is: \"\""), ""); + } + + #[test] + fn translation_and_explicit_context_keep_audio_marker_last() { + let mut request = audio_request(); + request.language = Some("German".to_string()); + request.prompt = Some("mesh-llm".to_string()); + assert_eq!( + audio_text_instruction(&request, false), + "Transcribe audio to text (language: German) Use this context when resolving names or terminology: mesh-llm" + ); + + let instruction = audio_text_instruction(&request, true); + assert!(instruction.starts_with("Translate the supplied audio into English.")); + let message = audio_text_user_message(instruction); + let Some(MessageContent::Parts(parts)) = message.content else { + panic!("audio translation must use multipart chat content"); + }; + assert_eq!(parts[0].content_type, "text"); + assert_eq!(parts[1].content_type, "input_audio"); + } + + #[test] + fn speech_voice_rejects_unsupported_speakers_without_reinterpreting_language() { + validate_speech_voice("default").expect("the default native speaker is supported"); + + for voice in ["alloy", "english", ""] { + let error = validate_speech_voice(voice).expect_err("speaker selection is unsupported"); + assert_eq!( + error.body().error.code.as_deref(), + Some("unsupported_model_feature") + ); + assert_eq!(error.body().error.param.as_deref(), Some("voice")); + assert!(error.body().error.message.contains("voice 'default'")); + } + } + + #[test] + fn workload_batch_stops_before_the_next_native_call_after_cancellation() { + let cancellation = openai_frontend::CancellationToken::new(); + let mut calls = 0; + let result = collect_workload_batch(0..3, &cancellation, |item| { + calls += 1; + cancellation.cancel(); + Ok(item) + }); + let error = result.expect_err("the remaining batch must stop after cancellation"); + assert_eq!(calls, 1); + assert_eq!( + workload_error(error).body().error.code.as_deref(), + Some("request_cancelled") + ); + } + + #[test] + fn workload_error_keeps_structured_cancellation() { + let error = workload_error(anyhow::Error::new(super::request_cancelled_error())); + assert_eq!( + error.body().error.code.as_deref(), + Some("request_cancelled") + ); + } +} diff --git a/crates/skippy-server/src/frontend/backend/tests.rs b/crates/skippy-server/src/frontend/backend/tests.rs index e1c0afd50c..82ee96cdaa 100644 --- a/crates/skippy-server/src/frontend/backend/tests.rs +++ b/crates/skippy-server/src/frontend/backend/tests.rs @@ -5,6 +5,7 @@ use crate::frontend::admission::GenerationTokenBudget; use crate::frontend::generation::ADMISSION_STARVATION_BOUND_TURNS; use crate::frontend::generation::OpenAiBackendMode; use crate::frontend::iteration_scheduler::IterationScheduler; +use crate::frontend::prefill::PrefillChunkPolicy; use crate::runtime_state::RuntimeState; use futures_util::StreamExt; use openai_frontend::ChatCompletionChunk; @@ -1246,6 +1247,75 @@ fn hooks_test_backend(hook_policy: Option>) -> StageOp } } +fn embedded_non_chat_test_mode(config: skippy_protocol::StageConfig) -> OpenAiBackendMode { + OpenAiBackendMode::EmbeddedStageZero { + config, + prefill_chunk_policy: PrefillChunkPolicy::Fixed { chunk_size: 64 }, + activation_width: 0, + downstream_wire_condition: crate::binary_transport::WireCondition::new(0.0, None) + .expect("unconditioned test wire"), + prefill_reply_credit_limit: 0, + lane_pool: None, + prediction_returns: None, + } +} + +#[test] +fn embedded_stage_zero_admits_unsplit_local_non_chat_topology() { + let mut backend = hooks_test_backend(None); + backend.mode = embedded_non_chat_test_mode(backend.config.clone()); + + assert!(backend.has_unsplit_full_model_topology()); +} + +#[test] +fn non_chat_topology_guard_rejects_staged_and_filtered_models() { + let mut backend = hooks_test_backend(None); + let full_config = backend.config.clone(); + let mut cases = Vec::new(); + + let mut downstream = full_config.clone(); + downstream.downstream = Some(skippy_protocol::PeerConfig { + stage_id: "stage-1".to_string(), + stage_index: 1, + endpoint: "127.0.0.1:0".to_string(), + }); + cases.push(downstream); + + let mut upstream = full_config.clone(); + upstream.upstream = Some(skippy_protocol::PeerConfig { + stage_id: "stage-previous".to_string(), + stage_index: 0, + endpoint: "127.0.0.1:0".to_string(), + }); + cases.push(upstream); + + let mut filtered = full_config.clone(); + filtered.filter_tensors_on_load = true; + cases.push(filtered); + + let mut partial = full_config.clone(); + partial.layer_start = 1; + cases.push(partial); + + let mut non_first_stage = full_config.clone(); + non_first_stage.stage_index = 1; + cases.push(non_first_stage); + + let mut empty = full_config.clone(); + empty.layer_end = 0; + cases.push(empty); + + for config in cases { + backend.mode = embedded_non_chat_test_mode(config.clone()); + assert!(!backend.has_unsplit_full_model_topology(), "{config:?}"); + backend.mode = OpenAiBackendMode::LocalRuntime; + backend.config = config; + assert!(!backend.has_unsplit_full_model_topology()); + backend.config = full_config.clone(); + } +} + fn mesh_hooks_request(model: &str) -> ChatCompletionRequest { let mut request: ChatCompletionRequest = serde_json::from_value(json!({ "model": model, diff --git a/crates/skippy-server/src/frontend/generation_flow.rs b/crates/skippy-server/src/frontend/generation_flow.rs index e190190117..72b42b1509 100644 --- a/crates/skippy-server/src/frontend/generation_flow.rs +++ b/crates/skippy-server/src/frontend/generation_flow.rs @@ -1,3 +1,4 @@ +mod encoder_decoder; mod text_generation; use crate::binary_transport::forwarded_stage_message_timed; diff --git a/crates/skippy-server/src/frontend/generation_flow/encoder_decoder.rs b/crates/skippy-server/src/frontend/generation_flow/encoder_decoder.rs new file mode 100644 index 0000000000..cac379dcf1 --- /dev/null +++ b/crates/skippy-server/src/frontend/generation_flow/encoder_decoder.rs @@ -0,0 +1,95 @@ +use std::time::Instant; + +use openai_frontend::{ChatCompletionRequest, OpenAiError, OpenAiResult}; +use skippy_runtime::{ModelWorkload, SamplingConfig}; + +use crate::frontend::generation::{ + GenerationCacheStats, OpenAiGenerationIds, StageOpenAiBackend, TokenControl, + tool_calls_requested, +}; +use crate::frontend::util::openai_backend_error; + +use super::LocalSessionCleanupGuard; + +impl StageOpenAiBackend { + #[allow(clippy::too_many_arguments)] + pub(super) fn generate_encoder_decoder_tokens( + &self, + prompt_token_ids: &[i32], + max_tokens: u32, + sampling: &SamplingConfig, + chat_request: Option<&ChatCompletionRequest>, + cancellation: Option<&openai_frontend::CancellationToken>, + ids: &OpenAiGenerationIds, + mut emit_token: impl FnMut(i32) -> OpenAiResult, + ) -> OpenAiResult { + if !self.has_unsplit_full_model_topology() { + return Err(OpenAiError::unsupported( + "encoder-decoder models currently require an unsplit local runtime", + )); + } + self.ensure_local_workload(ModelWorkload::EncoderDecoder)?; + if chat_request.is_some_and(tool_calls_requested) { + return Err(OpenAiError::unsupported( + "tool calls are not supported by encoder-decoder models", + )); + } + let session_id = ids.session_label.clone(); + let (result, mut cleanup) = LocalSessionCleanupGuard::run( + || self.cleanup_local_generation_session(&session_id, ids), + || { + let mut runtime = self + .runtime + .lock() + .map_err(|_| OpenAiError::backend("runtime lock poisoned"))?; + let prompt_started = Instant::now(); + let decoder_start = runtime + .encode_prompt(&session_id, prompt_token_ids) + .map_err(openai_backend_error)?; + let prompt_ms = prompt_started.elapsed().as_secs_f64() * 1_000.0; + let predicted_started = Instant::now(); + if max_tokens > 0 { + let mut predicted = runtime + .decode_frame_sampled( + &session_id, + decoder_start, + sampling.enabled.then_some(sampling), + None, + 0, + ) + .map_err(openai_backend_error)? + .0; + for generated in 0..max_tokens { + if cancellation + .is_some_and(openai_frontend::CancellationToken::is_cancelled) + { + return Err(OpenAiError::backend("request cancelled")); + } + if emit_token(predicted)? == TokenControl::Stop { + break; + } + if generated + 1 < max_tokens { + predicted = runtime + .decode_frame_sampled( + &session_id, + predicted, + sampling.enabled.then_some(sampling), + None, + 0, + ) + .map_err(openai_backend_error)? + .0; + } + } + } + Ok(GenerationCacheStats { + prompt_ms, + predicted_ms: predicted_started.elapsed().as_secs_f64() * 1_000.0, + ..GenerationCacheStats::default() + }) + }, + ); + cleanup.cleanup(); + result + } +} diff --git a/crates/skippy-server/src/frontend/generation_flow/text_generation.rs b/crates/skippy-server/src/frontend/generation_flow/text_generation.rs index 02d28d694d..e347f6f322 100644 --- a/crates/skippy-server/src/frontend/generation_flow/text_generation.rs +++ b/crates/skippy-server/src/frontend/generation_flow/text_generation.rs @@ -6,7 +6,7 @@ use crate::frontend::generation::{ use crate::frontend::util::{generation_stop_values, openai_backend_error}; use openai_frontend::{ChatCompletionRequest, OpenAiError, OpenAiResult}; use serde_json::json; -use skippy_runtime::SamplingConfig; +use skippy_runtime::{ModelWorkload, SamplingConfig}; pub(super) fn resident_capacity_target_tokens(prompt_token_count: usize) -> u64 { u64::try_from(prompt_token_count).unwrap_or(u64::MAX) @@ -91,6 +91,33 @@ impl StageOpenAiBackend { Some(prepared) => prepared, None => self.prepare_text_prompt(&prompt, max_tokens, &ids)?, }; + let workload = self + .runtime + .lock() + .map_err(|_| OpenAiError::backend("runtime lock poisoned"))? + .workload_info() + .map_err(openai_backend_error)? + .kind; + if workload == ModelWorkload::EncoderDecoder { + let mut collector = + TextGenerationCollector::new(self.runtime.clone(), stop_values, on_text_chunk)? + .with_ignore_eos(sampling.ignore_eos); + let cache_stats = self.generate_encoder_decoder_tokens( + &prompt_token_ids, + max_tokens, + &sampling, + hook_request.as_ref(), + cancellation, + &ids, + |token| collector.push_token(token), + )?; + return collector.finish(prompt_token_ids.len(), cache_stats); + } + if workload != ModelWorkload::CausalGeneration { + return Err(OpenAiError::unsupported(format!( + "model workload is {workload:?}; chat and completion endpoints require a generative model" + ))); + } // This is an optional cache candidate. The already-rendered prompt is // valid even when its second, assistant-marker-free rendering cannot // be tokenized, so bypass the candidate rather than failing the chat diff --git a/crates/skippy-server/src/frontend/tests/mod.rs b/crates/skippy-server/src/frontend/tests/mod.rs index d9c8adaabd..4507234f2d 100644 --- a/crates/skippy-server/src/frontend/tests/mod.rs +++ b/crates/skippy-server/src/frontend/tests/mod.rs @@ -11,7 +11,7 @@ pub(super) use super::{ pub(super) use crate::binary_transport::WireCondition; pub(super) use crate::kv_integration::PrefillKvIdentity; pub(super) use crate::kv_integration::{KvStageIntegration, proactive_eviction_attrs}; -pub(super) use crate::runtime_state::load_runtime; +pub(super) use crate::runtime_state::{load_runtime, reject_unsupported_staged_workload}; pub(super) use crate::telemetry::Telemetry; pub(super) use anyhow::{Context as _, Result, anyhow, bail}; pub(super) use async_trait::async_trait; @@ -27,8 +27,8 @@ pub(super) use openai_frontend::{ pub(super) use serde_json::{Value, json}; pub(super) use skippy_metrics::attr as attr_key; pub(super) use skippy_protocol::{ - LoadMode, MessageBase, PeerConfig, SCHEMA_VERSION, StageConfig, StageKvCacheConfig, - StageKvCacheMode, StageKvCachePayload, + LoadMode, MessageBase, PeerConfig, SCHEMA_VERSION, StageConfig, StageDevice, + StageKvCacheConfig, StageKvCacheMode, StageKvCachePayload, binary::{LLAMA_TOKEN_NULL, StageReplyStats, WireMessageKind, write_stage_message}, }; pub(super) use skippy_runtime::{ @@ -52,9 +52,11 @@ mod draft_runtime; mod generation; mod guardrails; mod multimodal; +mod non_chat; mod prefill; mod prefix_cache; mod prompting; mod request; mod support; +mod tts_oracle; mod wire_messages; diff --git a/crates/skippy-server/src/frontend/tests/multimodal.rs b/crates/skippy-server/src/frontend/tests/multimodal.rs index eee7d48e87..3c97e78d2e 100644 --- a/crates/skippy-server/src/frontend/tests/multimodal.rs +++ b/crates/skippy-server/src/frontend/tests/multimodal.rs @@ -191,46 +191,6 @@ fn multimodal_stage_config( } } -fn local_openai_backend(config: StageConfig) -> Result { - let runtime = load_runtime(&config)?.context("load smoke runtime")?; - let ctx_size = usize::try_from(config.ctx_size).unwrap_or(usize::MAX); - let telemetry = Telemetry::new( - None, - 1, - config.clone(), - crate::telemetry::TelemetryLevel::Off, - ); - let iteration_scheduler = - IterationScheduler::new(runtime.clone(), &config, 1, true, telemetry.clone())?; - Ok(StageOpenAiBackend { - runtime, - telemetry, - config, - model_id: "mm-smoke".to_string(), - default_max_tokens: 16, - request_defaults: EmbeddedOpenAiRequestDefaults::default(), - ctx_size, - mode: OpenAiBackendMode::LocalRuntime, - draft: None, - speculative_window: 0, - adaptive_speculative_window: false, - ngram_max: 0, - speculative: SpeculativeDecodeConfig::default(), - generation_limit: Arc::new(GenerationConcurrencyController::fixed(1)), - generation_queue_depth: Arc::new(AtomicUsize::new(0)), - generation_queue_limit: 1, - generation_admission_timeout: std::time::Duration::from_secs(10), - generation_service_estimator: Arc::new(crate::frontend::GenerationServiceEstimator::new(1)), - generation_session_locks: Arc::new(Mutex::new(std::collections::BTreeMap::new())), - generation_token_budget: Arc::new(GenerationTokenBudget::new(ctx_size)), - hook_policy: None, - generation_receipt: None, - linear_proposal_ingress: None, - kv: None, - iteration_scheduler, - }) -} - fn multimodal_chat_request(fixture: &MultimodalSmokeFixture) -> Result { multimodal_chat_request_with_max_tokens(fixture, fixture.max_tokens) } @@ -333,7 +293,7 @@ async fn real_multimodal_local_smoke_when_fixture_is_set() -> Result<()> { fixture.layer_end, available_loopback_addr()?, ); - let backend = local_openai_backend(config)?; + let backend = support::local_openai_backend(config, "mm-smoke")?; let response = backend .chat_completion(multimodal_chat_request(&fixture)?) .await?; @@ -355,7 +315,7 @@ async fn real_multimodal_local_prefill_failure_releases_lane_when_fixture_is_set fixture.layer_end, available_loopback_addr()?, ); - let backend = local_openai_backend(config)?; + let backend = support::local_openai_backend(config, "mm-smoke")?; for attempt in 0..2 { let result = backend @@ -398,7 +358,7 @@ async fn real_multimodal_length_limit_then_next_image_uses_clean_lane_when_fixtu available_loopback_addr()?, ); config.lane_count = 2; - let backend = local_openai_backend(config)?; + let backend = support::local_openai_backend(config, "mm-smoke")?; backend .runtime .lock() diff --git a/crates/skippy-server/src/frontend/tests/non_chat.rs b/crates/skippy-server/src/frontend/tests/non_chat.rs new file mode 100644 index 0000000000..1327d75c87 --- /dev/null +++ b/crates/skippy-server/src/frontend/tests/non_chat.rs @@ -0,0 +1,424 @@ +use super::*; + +use openai_frontend::{ + AudioFormat, AudioSpeechRequest, AudioTranscriptionRequest, EmbeddingInput, EmbeddingOutput, + EmbeddingsRequest, RerankDocument, RerankRequest, +}; +use skippy_runtime::ModelWorkload; + +const MODEL_ENV: &str = "SKIPPY_WORKLOAD_MODEL"; +const MODEL_ID_ENV: &str = "SKIPPY_WORKLOAD_MODEL_ID"; +const CLASS_ENV: &str = "SKIPPY_WORKLOAD_CLASS"; +const PROJECTOR_ENV: &str = "SKIPPY_WORKLOAD_PROJECTOR"; +const MEDIA_ENV: &str = "SKIPPY_WORKLOAD_MEDIA"; +const LAYER_END_ENV: &str = "SKIPPY_WORKLOAD_LAYER_END"; +const CTX_SIZE_ENV: &str = "SKIPPY_WORKLOAD_CTX_SIZE"; +const MAX_TOKENS_ENV: &str = "SKIPPY_WORKLOAD_MAX_TOKENS"; +const N_GPU_LAYERS_ENV: &str = "SKIPPY_WORKLOAD_N_GPU_LAYERS"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum CertifiedWorkloadClass { + Embedding, + Rerank, + EncoderDecoder, + Ocr, + SpeechSynthesis, + SpeechRecognition, +} + +impl CertifiedWorkloadClass { + fn parse(value: &str) -> Result { + match value { + "embedding" => Ok(Self::Embedding), + "rerank" => Ok(Self::Rerank), + "encoder_decoder" => Ok(Self::EncoderDecoder), + "ocr" => Ok(Self::Ocr), + "speech_synthesis" => Ok(Self::SpeechSynthesis), + "speech_recognition" => Ok(Self::SpeechRecognition), + other => bail!("unsupported {CLASS_ENV} value {other:?}"), + } + } + + fn requires_projector(self) -> bool { + matches!( + self, + Self::Ocr | Self::SpeechSynthesis | Self::SpeechRecognition + ) + } + + fn requires_media(self) -> bool { + matches!(self, Self::Ocr | Self::SpeechRecognition) + } + + fn staging_label(self) -> Option<&'static str> { + match self { + Self::Embedding => Some("embedding"), + Self::Rerank => Some("rerank"), + Self::EncoderDecoder => Some("encoder_decoder"), + Self::SpeechSynthesis => Some("speech_synthesis"), + Self::Ocr | Self::SpeechRecognition => None, + } + } +} + +struct WorkloadFixture { + class: CertifiedWorkloadClass, + model_id: String, + model_path: PathBuf, + projector_path: Option, + media_path: Option, + layer_end: u32, + ctx_size: u32, + max_tokens: u32, + n_gpu_layers: i32, +} + +fn required_file(name: &str) -> Result { + let path = PathBuf::from(env::var_os(name).context(format!("{name} is required"))?); + if !path.is_file() { + bail!("{name} does not point at a file: {}", path.display()); + } + Ok(path) +} + +fn parse_env(name: &str, default: T) -> Result +where + T: std::str::FromStr, + T::Err: std::fmt::Display, +{ + env::var(name).map_or(Ok(default), |value| { + value + .parse::() + .map_err(|error| anyhow!("parse {name}={value:?}: {error}")) + }) +} + +fn workload_fixture() -> Result> { + let Some(class) = env::var(CLASS_ENV).ok() else { + return Ok(None); + }; + let class = CertifiedWorkloadClass::parse(&class)?; + let projector_path = class + .requires_projector() + .then(|| required_file(PROJECTOR_ENV)) + .transpose()?; + let media_path = class + .requires_media() + .then(|| required_file(MEDIA_ENV)) + .transpose()?; + let model_path = required_file(MODEL_ENV)?; + let layer_end = parse_env(LAYER_END_ENV, 1_u32)?; + if layer_end == 0 { + bail!("{LAYER_END_ENV} must be positive"); + } + Ok(Some(WorkloadFixture { + class, + model_id: env::var(MODEL_ID_ENV).unwrap_or_else(|_| "workload-smoke".to_string()), + model_path, + projector_path, + media_path, + layer_end, + ctx_size: parse_env(CTX_SIZE_ENV, 2048)?, + max_tokens: parse_env(MAX_TOKENS_ENV, 32)?, + n_gpu_layers: parse_env(N_GPU_LAYERS_ENV, 0)?, + })) +} + +fn workload_stage_config(fixture: &WorkloadFixture) -> StageConfig { + StageConfig { + run_id: "workload-certification".to_string(), + topology_id: "workload-certification-local".to_string(), + model_id: fixture.model_id.clone(), + model_path: Some(fixture.model_path.to_string_lossy().to_string()), + projector_path: fixture + .projector_path + .as_ref() + .map(|path| path.to_string_lossy().to_string()), + stage_id: "stage-0".to_string(), + stage_index: 0, + layer_start: 0, + layer_end: fixture.layer_end, + ctx_size: fixture.ctx_size, + lane_count: 1, + n_batch: Some(fixture.ctx_size.min(2048)), + n_ubatch: Some(fixture.ctx_size.min(2048)), + n_gpu_layers: fixture.n_gpu_layers, + kv_offload: (fixture.n_gpu_layers == 0).then_some(false), + op_offload: (fixture.n_gpu_layers == 0).then_some(false), + selected_device: (fixture.n_gpu_layers == 0).then(|| StageDevice { + backend_device: "CPU".to_string(), + stable_id: None, + index: None, + vram_bytes: None, + }), + filter_tensors_on_load: false, + native_mtp_enabled: false, + load_mode: LoadMode::RuntimeSlice, + bind_addr: "127.0.0.1:0".to_string(), + ..StageConfig::default() + } +} + +fn assert_vectors_close(left: &[f32], right: &[f32]) { + assert_eq!(left.len(), right.len()); + let maximum_delta = left + .iter() + .zip(right) + .map(|(left, right)| (left - right).abs()) + .fold(0.0_f32, f32::max); + assert!(maximum_delta <= 1e-5, "embedding delta {maximum_delta}"); +} + +async fn certify_embedding(backend: &StageOpenAiBackend) -> Result<()> { + let info = backend.ensure_local_workload(ModelWorkload::Embedding)?; + assert!(info.output_dimensions > 0); + let request = EmbeddingsRequest { + model: backend.model_id.clone(), + input: EmbeddingInput::Texts(vec![ + "search_query: distributed inference".to_string(), + "search_document: GPUs collaborate over a mesh".to_string(), + ]), + encoding_format: "float".to_string(), + dimensions: None, + user: None, + }; + let first = backend + .embeddings(request.clone(), OpenAiRequestContext::new()) + .await?; + let second = backend + .embeddings(request, OpenAiRequestContext::new()) + .await?; + assert_eq!(first.object, "list"); + assert_eq!(first.data.len(), 2); + assert!(first.usage.prompt_tokens > 0); + for (left, right) in first.data.iter().zip(&second.data) { + let (EmbeddingOutput::Float(left), EmbeddingOutput::Float(right)) = + (&left.embedding, &right.embedding) + else { + bail!("float embedding request returned a non-float payload"); + }; + assert_eq!(left.len(), info.output_dimensions as usize); + let norm = left.iter().map(|value| value * value).sum::().sqrt(); + assert!((norm - 1.0).abs() <= 1e-4, "embedding norm {norm}"); + assert_vectors_close(left, right); + } + Ok(()) +} + +async fn certify_rerank(backend: &StageOpenAiBackend) -> Result<()> { + let info = backend.ensure_local_workload(ModelWorkload::Rerank)?; + assert_eq!(info.classifier_outputs, 1); + let request = RerankRequest { + model: backend.model_id.clone(), + query: "distributed GPU inference".to_string(), + documents: vec![ + RerankDocument::Text("GPUs share one language model over a mesh".to_string()), + RerankDocument::Text("A recipe for tomato soup".to_string()), + ], + top_n: None, + return_documents: true, + }; + let first = backend + .rerank(request.clone(), OpenAiRequestContext::new()) + .await?; + let second = backend.rerank(request, OpenAiRequestContext::new()).await?; + assert_eq!(first.results.len(), 2); + assert!(first.usage.prompt_tokens > 0); + for (left, right) in first.results.iter().zip(&second.results) { + assert_eq!(left.index, right.index); + assert!(left.relevance_score.is_finite()); + assert!((left.relevance_score - right.relevance_score).abs() <= 1e-6); + assert!(left.document.is_some()); + } + Ok(()) +} + +async fn certify_encoder_decoder(backend: &StageOpenAiBackend, max_tokens: u32) -> Result<()> { + backend.ensure_local_workload(ModelWorkload::EncoderDecoder)?; + let request: CompletionRequest = serde_json::from_value(json!({ + "model": backend.model_id, + "prompt": "translate English to German: The house is wonderful.", + "max_tokens": max_tokens, + "temperature": 0.0 + }))?; + let first = backend.completion(request.clone()).await?; + let second = backend.completion(request).await?; + assert!(!first.choices[0].text.trim().is_empty()); + assert_eq!(first.choices[0].text, second.choices[0].text); + assert!(first.usage.prompt_tokens > 0); + assert!(first.usage.completion_tokens > 0); + Ok(()) +} + +fn media_chat_request(fixture: &WorkloadFixture) -> Result { + let path = fixture + .media_path + .as_ref() + .context("media path is required")?; + let encoded = base64::engine::general_purpose::STANDARD.encode(fs::read(path)?); + serde_json::from_value(json!({ + "model": fixture.model_id, + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": "Read all visible text. Return only the transcription."}, + {"type": "image_url", "image_url": {"url": format!("data:image/png;base64,{encoded}")}} + ] + }], + "max_tokens": fixture.max_tokens, + "temperature": 0.0 + })) + .context("build OCR request") +} + +async fn certify_ocr(backend: &StageOpenAiBackend, fixture: &WorkloadFixture) -> Result<()> { + let first = backend + .chat_completion(media_chat_request(fixture)?) + .await?; + let second = backend + .chat_completion(media_chat_request(fixture)?) + .await?; + let first_text = first.choices[0] + .message + .content + .as_deref() + .unwrap_or_default() + .trim(); + let second_text = second.choices[0] + .message + .content + .as_deref() + .unwrap_or_default() + .trim(); + assert!(!first_text.is_empty()); + assert_eq!(first_text, second_text); + Ok(()) +} + +async fn certify_speech_synthesis(backend: &StageOpenAiBackend) -> Result<()> { + assert!( + backend + .runtime + .lock() + .expect("runtime mutex poisoned") + .supports_speech_synthesis() + ); + let unsupported_voice = backend + .audio_speech( + AudioSpeechRequest { + model: backend.model_id.clone(), + input: "The mesh is ready.".to_string(), + voice: "alloy".to_string(), + response_format: AudioFormat::Wav, + speed: 1.0, + }, + OpenAiRequestContext::new(), + ) + .await + .expect_err("speaker selection must not be interpreted as a language"); + assert_eq!( + unsupported_voice.body().error.code.as_deref(), + Some("unsupported_model_feature") + ); + assert_eq!( + unsupported_voice.body().error.param.as_deref(), + Some("voice") + ); + let response = backend + .audio_speech( + AudioSpeechRequest { + model: backend.model_id.clone(), + input: "The mesh is ready.".to_string(), + voice: "default".to_string(), + response_format: AudioFormat::Wav, + speed: 1.0, + }, + OpenAiRequestContext::new(), + ) + .await?; + assert_eq!(response.content_type, "audio/wav"); + assert!(response.bytes.len() > 44); + assert_eq!(&response.bytes[..4], b"RIFF"); + assert_eq!(&response.bytes[8..12], b"WAVE"); + Ok(()) +} + +async fn certify_speech_recognition( + backend: &StageOpenAiBackend, + fixture: &WorkloadFixture, +) -> Result<()> { + let media_path = fixture + .media_path + .as_ref() + .context("media path is required")?; + let response = backend + .audio_transcription( + AudioTranscriptionRequest { + model: fixture.model_id.clone(), + file: fs::read(media_path)?, + filename: media_path + .file_name() + .map(|name| name.to_string_lossy().to_string()), + language: Some("en".to_string()), + prompt: None, + response_format: "json".to_string(), + temperature: Some(0.0), + }, + OpenAiRequestContext::new(), + ) + .await?; + assert!(!response.text.trim().is_empty()); + Ok(()) +} + +fn assert_unsupported_staging( + backend: &StageOpenAiBackend, + fixture: &WorkloadFixture, + expected: &str, +) -> Result<()> { + let mut config = workload_stage_config(fixture); + config.filter_tensors_on_load = true; + config.layer_end = (fixture.layer_end / 2).max(1); + config.downstream = Some(PeerConfig { + stage_id: "stage-1".to_string(), + stage_index: 1, + endpoint: "127.0.0.1:1".to_string(), + }); + let runtime = backend.runtime.lock().expect("runtime mutex poisoned"); + let error = match reject_unsupported_staged_workload(&config, &runtime.model) { + Ok(()) => bail!("unsupported workload staging did not fail closed"), + Err(error) => error, + }; + let rendered = format!("{error:#}"); + assert!( + rendered.contains("unsupported staged workload") && rendered.contains(expected), + "unexpected staged workload error: {rendered}" + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn real_non_chat_class_smoke_when_fixture_is_set() -> Result<()> { + let Some(fixture) = workload_fixture()? else { + return Ok(()); + }; + let backend = + support::local_openai_backend(workload_stage_config(&fixture), fixture.model_id.clone())?; + match fixture.class { + CertifiedWorkloadClass::Embedding => certify_embedding(&backend).await?, + CertifiedWorkloadClass::Rerank => certify_rerank(&backend).await?, + CertifiedWorkloadClass::EncoderDecoder => { + certify_encoder_decoder(&backend, fixture.max_tokens).await? + } + CertifiedWorkloadClass::Ocr => certify_ocr(&backend, &fixture).await?, + CertifiedWorkloadClass::SpeechSynthesis => certify_speech_synthesis(&backend).await?, + CertifiedWorkloadClass::SpeechRecognition => { + certify_speech_recognition(&backend, &fixture).await? + } + } + if let Some(staging_label) = fixture.class.staging_label() { + assert_unsupported_staging(&backend, &fixture, staging_label)?; + } + Ok(()) +} diff --git a/crates/skippy-server/src/frontend/tests/support.rs b/crates/skippy-server/src/frontend/tests/support.rs index 1de05d1a68..2562d975f2 100644 --- a/crates/skippy-server/src/frontend/tests/support.rs +++ b/crates/skippy-server/src/frontend/tests/support.rs @@ -117,6 +117,49 @@ pub(super) fn unsupported_code(error: OpenAiError) -> Option { error.body().error.code } +pub(super) fn local_openai_backend( + config: StageConfig, + model_id: impl Into, +) -> Result { + let runtime = load_runtime(&config)?.context("load local test runtime")?; + let ctx_size = usize::try_from(config.ctx_size).unwrap_or(usize::MAX); + let telemetry = Telemetry::new( + None, + 1, + config.clone(), + crate::telemetry::TelemetryLevel::Off, + ); + let iteration_scheduler = + IterationScheduler::new(runtime.clone(), &config, 1, true, telemetry.clone())?; + Ok(StageOpenAiBackend { + runtime, + telemetry, + config, + model_id: model_id.into(), + default_max_tokens: 16, + request_defaults: EmbeddedOpenAiRequestDefaults::default(), + ctx_size, + mode: OpenAiBackendMode::LocalRuntime, + draft: None, + speculative_window: 0, + adaptive_speculative_window: false, + ngram_max: 0, + speculative: SpeculativeDecodeConfig::default(), + generation_limit: Arc::new(GenerationConcurrencyController::fixed(1)), + generation_queue_depth: Arc::new(AtomicUsize::new(0)), + generation_queue_limit: 1, + generation_admission_timeout: Duration::from_secs(10), + generation_service_estimator: Arc::new(GenerationServiceEstimator::new(1)), + generation_session_locks: Arc::new(Mutex::new(std::collections::BTreeMap::new())), + generation_token_budget: Arc::new(GenerationTokenBudget::new(ctx_size)), + hook_policy: None, + generation_receipt: None, + linear_proposal_ingress: None, + kv: None, + iteration_scheduler, + }) +} + pub(super) fn test_request_defaults() -> EmbeddedOpenAiRequestDefaults { EmbeddedOpenAiRequestDefaults { stop: Some(vec!["".to_string()]), diff --git a/crates/skippy-server/src/frontend/tests/tts_oracle.rs b/crates/skippy-server/src/frontend/tests/tts_oracle.rs new file mode 100644 index 0000000000..2bf3c8646e --- /dev/null +++ b/crates/skippy-server/src/frontend/tests/tts_oracle.rs @@ -0,0 +1,116 @@ +//! Deterministic test-only TTS output for comparison with pinned llama-tts. +//! +//! The public speech endpoint deliberately uses a random seed. An independent +//! waveform oracle therefore has to drive the same local model execution with +//! fixed sampling parameters rather than compare two random HTTP responses. + +use super::*; +use skippy_runtime::{SpeechOutputFormat, SpeechSynthesisConfig}; + +const OUTPUT_ENV: &str = "SKIPPY_TTS_ORACLE_CANDIDATE_WAV"; + +fn fixture_path(name: &str) -> Result { + let path = PathBuf::from(env::var_os(name).context(format!("{name} is required"))?); + if !path.is_file() { + bail!("{name} does not point at a file: {}", path.display()); + } + Ok(path) +} + +fn fixture_number(name: &str) -> Result +where + T: std::str::FromStr, + T::Err: std::fmt::Display, +{ + env::var(name) + .with_context(|| format!("{name} is required"))? + .parse::() + .map_err(|error| anyhow!("parse {name}: {error}")) +} + +fn local_tts_config( + model_id: &str, + model_path: &Path, + projector_path: &Path, + layer_end: u32, +) -> StageConfig { + StageConfig { + run_id: "tts-monolithic-oracle".to_string(), + topology_id: "tts-monolithic-oracle-local".to_string(), + model_id: model_id.to_string(), + model_path: Some(model_path.to_string_lossy().to_string()), + projector_path: Some(projector_path.to_string_lossy().to_string()), + stage_id: "stage-0".to_string(), + stage_index: 0, + layer_start: 0, + layer_end, + ctx_size: 2048, + lane_count: 1, + n_batch: Some(2048), + n_ubatch: Some(2048), + n_gpu_layers: 0, + kv_offload: Some(false), + op_offload: Some(false), + selected_device: Some(StageDevice { + backend_device: "CPU".to_string(), + stable_id: None, + index: None, + vram_bytes: None, + }), + filter_tensors_on_load: false, + native_mtp_enabled: false, + load_mode: LoadMode::RuntimeSlice, + bind_addr: "127.0.0.1:0".to_string(), + ..StageConfig::default() + } +} + +#[test] +fn deterministic_tts_candidate_when_fixture_is_set() -> Result<()> { + let Some(output_path) = env::var_os(OUTPUT_ENV) else { + return Ok(()); + }; + let output_path = PathBuf::from(output_path); + let model_path = fixture_path("SKIPPY_WORKLOAD_MODEL")?; + let projector_path = fixture_path("SKIPPY_WORKLOAD_PROJECTOR")?; + let model_id = + env::var("SKIPPY_WORKLOAD_MODEL_ID").context("SKIPPY_WORKLOAD_MODEL_ID is required")?; + let layer_end: u32 = fixture_number("SKIPPY_WORKLOAD_LAYER_END")?; + if layer_end == 0 { + bail!("SKIPPY_WORKLOAD_LAYER_END must be positive"); + } + let prompt = + env::var("SKIPPY_TTS_ORACLE_PROMPT").context("SKIPPY_TTS_ORACLE_PROMPT is required")?; + let seed: u32 = fixture_number("SKIPPY_TTS_ORACLE_SEED")?; + let top_k: i32 = fixture_number("SKIPPY_TTS_ORACLE_TOP_K")?; + let top_p: f32 = fixture_number("SKIPPY_TTS_ORACLE_TOP_P")?; + let max_frames: usize = fixture_number("SKIPPY_TTS_ORACLE_MAX_FRAMES")?; + if prompt.is_empty() || top_k < 1 || !(0.0..=1.0).contains(&top_p) || max_frames == 0 { + bail!("invalid deterministic TTS oracle parameters"); + } + + let config = local_tts_config(&model_id, &model_path, &projector_path, layer_end); + let runtime = load_runtime(&config)?.context("load full-model TTS oracle candidate")?; + let mut runtime = runtime + .lock() + .map_err(|_| anyhow!("TTS oracle candidate runtime lock poisoned"))?; + let audio = runtime.synthesize_speech( + "tts-oracle", + &SpeechSynthesisConfig { + prompt, + language: None, + top_k, + top_p, + seed, + output_format: SpeechOutputFormat::Wav, + max_frames, + }, + || false, + )?; + if audio.bytes.len() <= 44 || audio.sample_rate == 0 || audio.sample_count == 0 { + bail!("deterministic TTS candidate produced no WAV samples"); + } + fs::write(&output_path, &audio.bytes) + .with_context(|| format!("write candidate WAV to {}", output_path.display()))?; + Ok(()) +} diff --git a/crates/skippy-server/src/runtime_state.rs b/crates/skippy-server/src/runtime_state.rs index 23a8aa7689..787aefe87a 100644 --- a/crates/skippy-server/src/runtime_state.rs +++ b/crates/skippy-server/src/runtime_state.rs @@ -11,9 +11,10 @@ use skippy_runtime::{ DecodeFrameBatchRequest, FlashAttentionType as RuntimeFlashAttentionType, GenerationSignalWindow, GlmDsaPolicy as RuntimeGlmDsaPolicy, IterationBatchOutput, IterationBatchPhase, IterationBatchRequest, MediaInput, MediaPrefill, MediaPrefillFrame, - ModelStateKind, MtpSource, NativeMtpDraft, RuntimeConfig, RuntimeKvPage, RuntimeKvPageDesc, - RuntimeLoadMode, SamplingConfig, SplitMode as RuntimeSplitMode, StageModel, StageSession, - TokenSignal, parse_cache_type, + ModelStateKind, ModelWorkload, MtpSource, NativeMtpDraft, RuntimeConfig, RuntimeKvPage, + RuntimeKvPageDesc, RuntimeLoadMode, SamplingConfig, SpeechAudio, SpeechSynthesisConfig, + SplitMode as RuntimeSplitMode, StageModel, StageSession, TokenSignal, WorkloadInfo, + parse_cache_type, }; mod frame_operations; @@ -140,6 +141,44 @@ struct ResidentLanePrefix { } impl RuntimeState { + pub fn workload_info(&self) -> Result { + self.model.workload_info() + } + + pub fn embed( + &mut self, + session_id: &str, + token_ids: &[i32], + dimensions: usize, + ) -> Result> { + let embedding = self.session(session_id)?.embed(token_ids, dimensions)?; + self.session_token_counts.insert( + session_id.to_string(), + u64::try_from(token_ids.len()).context("embedding token count exceeds u64")?, + ); + Ok(embedding) + } + + pub fn rerank( + &mut self, + session_id: &str, + query: &str, + document: &str, + ) -> Result<(f32, usize)> { + let result = self.session(session_id)?.rerank(query, document)?; + self.session_token_counts.insert( + session_id.to_string(), + u64::try_from(result.1).context("rerank token count exceeds u64")?, + ); + Ok(result) + } + + pub fn encode_prompt(&mut self, session_id: &str, token_ids: &[i32]) -> Result { + let decoder_start = self.session(session_id)?.encode_prompt(token_ids)?; + self.session_token_counts.insert(session_id.to_string(), 0); + Ok(decoder_start) + } + pub fn input_activation_boundary(&self) -> Option { self.model.input_activation_boundary() } @@ -261,8 +300,15 @@ pub fn load_runtime_with_overrides( open_stage_model(model_path, &runtime_config)? } }; + Ok(Some(runtime_from_loaded_model(config, model)?)) +} - Ok(Some(Arc::new(Mutex::new(RuntimeState { +fn runtime_from_loaded_model( + config: &StageConfig, + model: StageModel, +) -> Result>> { + reject_unsupported_staged_workload(config, &model)?; + Ok(Arc::new(Mutex::new(RuntimeState { model, layer_start: config.layer_start, layer_end: config.layer_end, @@ -277,7 +323,7 @@ pub fn load_runtime_with_overrides( session_resident_prefixes: BTreeMap::new(), #[cfg(test)] modelless_for_test: false, - })))) + }))) } pub fn load_runtime_with_overrides_and_open_events( @@ -309,23 +355,34 @@ pub fn load_runtime_with_overrides_and_open_events( open_stage_model_with_events(model_path, &runtime_config, model_open_event_reporter)? } }; + Ok(Some(runtime_from_loaded_model(config, model)?)) +} - Ok(Some(Arc::new(Mutex::new(RuntimeState { - model, - layer_start: config.layer_start, - layer_end: config.layer_end, - lane_count: config.lane_count, - ctx_size: config.ctx_size, - next_lane_index: 0, - free_lane_indices: Vec::new(), - sessions: BTreeMap::new(), - idle_sessions: Vec::new(), - max_idle_sessions: max_idle_sessions_from_stage_config(config), - session_token_counts: BTreeMap::new(), - session_resident_prefixes: BTreeMap::new(), - #[cfg(test)] - modelless_for_test: false, - })))) +pub(crate) fn reject_unsupported_staged_workload( + config: &StageConfig, + model: &StageModel, +) -> Result<()> { + if !config.filter_tensors_on_load { + return Ok(()); + } + if model.supports_speech_synthesis() { + anyhow::bail!( + "unsupported staged workload speech_synthesis: audio generation requires an unsplit full model" + ); + } + let workload = model.workload_info()?.kind; + if workload != ModelWorkload::CausalGeneration { + anyhow::bail!( + "unsupported staged workload {}: non-chat execution requires an unsplit full model", + match workload { + ModelWorkload::CausalGeneration => unreachable!(), + ModelWorkload::Embedding => "embedding", + ModelWorkload::Rerank => "rerank", + ModelWorkload::EncoderDecoder => "encoder_decoder", + } + ); + } + Ok(()) } /// Translates `model_fit.cache_idle_slots` into the idle-session-pool bound. diff --git a/crates/skippy-server/src/runtime_state/frame_operations.rs b/crates/skippy-server/src/runtime_state/frame_operations.rs index 8155d5e72e..b5d77963ad 100644 --- a/crates/skippy-server/src/runtime_state/frame_operations.rs +++ b/crates/skippy-server/src/runtime_state/frame_operations.rs @@ -44,6 +44,23 @@ impl RuntimeState { self.model.has_media_projector() } + pub fn supports_speech_synthesis(&self) -> bool { + self.model.supports_speech_synthesis() + } + + pub fn synthesize_speech( + &mut self, + session_id: &str, + config: &SpeechSynthesisConfig, + cancellation_requested: impl Fn() -> bool, + ) -> Result { + let model = &self.model as *const StageModel; + let session = self.session(session_id)?; + // The outer RuntimeState mutex serializes both projector and session + // access; this splits borrows across those independently owned fields. + unsafe { (&*model).synthesize_speech(session, config, cancellation_requested) } + } + pub fn prefill_media( &mut self, session_id: &str, diff --git a/docs/NON_CHAT_MODELS.md b/docs/NON_CHAT_MODELS.md new file mode 100644 index 0000000000..05d6cc2a3f --- /dev/null +++ b/docs/NON_CHAT_MODELS.md @@ -0,0 +1,207 @@ +# Non-Chat Model Workloads + +Mesh LLM can serve several llama.cpp model classes whose primary contract is +not causal chat generation. The public OpenAI-compatible endpoint stays on the +normal mesh port (`http://127.0.0.1:9337/v1`), while the loaded model advertises +its runtime-probed workload class for safe routing. + +## Supported surfaces + +| Model class | HTTP surface | Current execution boundary | +|---|---|---| +| Embedding / encoder-only | `POST /v1/embeddings` | Unsplit local full model | +| Cross-encoder rerank | `POST /v1/rerank` | Unsplit local full model | +| Encoder-decoder | `POST /v1/completions`, `POST /v1/chat/completions` | Unsplit local full model | +| OCR | `POST /v1/chat/completions` or `POST /v1/responses` with an image input | Causal trunk and projector colocated on one node | +| Speech synthesis | `POST /v1/audio/speech` | Unsplit local full model and projector | +| Speech recognition / translation | `POST /v1/audio/transcriptions`, `POST /v1/audio/translations` | Causal trunk and audio projector colocated on one node | + +Embedding, rerank, encoder-decoder, and speech-synthesis workloads do not +silently enter the stage-split generation path. A filtered stage model returns +a structured `unsupported` error. OCR and speech recognition use causal trunks, +but their projector remains local to the trunk; they are not claims of a +distributed projector implementation. + +## Embeddings + +The request accepts a string, an array of strings, a token array, or an array of +token arrays. `encoding_format` may be `float` or `base64`. Base64 values encode +little-endian `f32` values. The runtime honors the pooling type recorded in the +GGUF and returns L2-normalized vectors. + +```python +from openai import OpenAI + +client = OpenAI(api_key="mesh", base_url="http://127.0.0.1:9337/v1") +response = client.embeddings.create( + model="nomic-embed-text-v1.5-Q8_0", + input=["search_query: distributed inference", "search_document: pooled GPUs"], + encoding_format="float", +) +print(len(response.data[0].embedding)) +``` + +The optional `dimensions` field is accepted only when it equals the model's +native output width. Dimensionality reduction is not performed implicitly. + +## Reranking + +`/v1/rerank` is the common cross-encoder companion surface. Documents may be +strings or objects containing a string `text` field. + +```bash +curl -sS http://127.0.0.1:9337/v1/rerank \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "jina-reranker-v1-turbo-en", + "query": "distributed model inference", + "documents": ["one GPU", "several GPUs connected over a mesh"], + "top_n": 2, + "return_documents": true + }' +``` + +Results are ordered by descending relevance score. Each result retains the +zero-based index of the original document. + +## Encoder-decoder generation and OCR + +Encoder-decoder GGUFs use the existing completion and chat response shapes. +Their encoder pass and decoder loop are executed by the local Skippy runtime; +tool calls are rejected because the current encoder-decoder path does not +implement that contract. + +OCR remains a multimodal text-generation request. Send the image using the +existing `image_url`/`input_image` content-part contract. Automatic routing +requires a model that advertises both a causal workload and vision capability. + +## Audio + +Speech synthesis returns raw binary data with the matching response content +type. The current native Qwen3-TTS path supports `wav` (`audio/wav`) and `pcm` +(`audio/pcm`). Other OpenAI response formats are parsed and rejected with a +structured unsupported error instead of returning bytes under the wrong media +type. Set `response_format` explicitly because the OpenAI request default is +`mp3`, which this native path does not encode. + +```bash +curl -sS http://127.0.0.1:9337/v1/audio/speech \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "Qwen3-TTS-12Hz-1.7B-Base-Q8_0", + "input": "The mesh is ready.", + "voice": "default", + "response_format": "wav" + }' \ + --output speech.wav +``` + +Transcription and translation accept `multipart/form-data` with a required +`model` field and one `file` field. Uploads are bounded at 64 MiB. The supported +response formats are `json` and `text`. + +```bash +curl -sS http://127.0.0.1:9337/v1/audio/transcriptions \ + -F model=ultravox-v0_5-llama-3_2-1b \ + -F file=@recording.wav \ + -F response_format=json +``` + +Automatic audio routing filters for a runtime-verified audio capability. An +explicitly selected incompatible model is rejected by the local backend rather +than silently discarding the audio. + +## Class-specific workload certification + +Every row in `ci/llama-canary/family-certified.json` records an explicit +`class`. The six checked-in non-chat rows use the certified `workload-oracle` +profile with separate class-specific smoke and local-monolithic oracle lanes; +they do not inherit the causal-generation `full` profile or its three split +handoff lanes. The checked-in representatives cover +embedding, rerank, encoder-decoder, OCR, speech synthesis, and speech recognition +with immutable Hugging Face revisions and artifact digests. + +Run a focused local smoke with the model files already present: + +```bash +scripts/skippy-workload-certify.sh \ + --class embedding \ + --lane embedding-smoke \ + --model-path /path/to/model.gguf \ + --model-id local-embedding \ + --work-dir target/workload-certification +``` + +Projector-backed classes additionally require `--projector-path`. Each lane +checks local real-model behavior and exercises its HTTP endpoint through the +OpenAI frontend. The embedding lane also uses the official Python OpenAI SDK +when that optional package is installed; its mandatory HTTP assertions do not +depend on the SDK being present. + +An isolated invocation without an oracle remains a smoke check only: it verifies +local execution, the HTTP response contract, and class-specific coarse +assertions, but cannot satisfy the checked-in certified profile. Filtered +staged workloads also fail closed where unsupported. The mandatory family +battery oracle uses test-only llama.cpp executables built from the same pinned +patch queue; those executables are never packaged as Mesh-LLM serving backends: + +```bash +LLAMA_STAGE_BACKEND=cpu \ +LLAMA_STAGE_LINK_MODE=static \ +LLAMA_STAGE_BUILD_DIR=/path/to/candidate-static-build \ +just llama-build + +LLAMA_STAGE_BACKEND=cpu \ +LLAMA_STAGE_LINK_MODE=static \ +LLAMA_STAGE_BUILD_DIR=/path/to/isolated-oracle-build \ +LLAMA_STAGE_WORKLOAD_ORACLE=ON \ +just llama-build + +LLAMA_STAGE_BACKEND=cpu \ +LLAMA_STAGE_LINK_MODE=static \ +LLAMA_STAGE_BUILD_DIR=/path/to/candidate-static-build \ +scripts/skippy-workload-certify.sh \ + --class embedding \ + --lane embedding-smoke \ + --model-path /path/to/model.gguf \ + --model-id local-embedding \ + --work-dir target/workload-certification \ + --oracle-server /path/to/isolated-oracle-build/bin/llama-server \ + --require-oracle +``` + +The runner checks each oracle executable's current pinned-source build stamp +and requires CPU-only execution, including Metal disabled on macOS. The +monolithic reference uses `--no-repack` to match the staged runtime's default +model-loading configuration; omitting that flag can create a numerical +mismatch unrelated to the workload implementation. Use +`--oracle-server` for embedding, rerank, OCR, or speech recognition; +`--oracle-completion` with `bin/llama-completion` for encoder-decoder; and +`--oracle-tts` with `bin/llama-tts` for speech synthesis. The family battery +accepts the corresponding `SKIPPY_WORKLOAD_ORACLE_SERVER`, +`SKIPPY_WORKLOAD_ORACLE_COMPLETION`, and `SKIPPY_WORKLOAD_ORACLE_TTS` +environment variables; a certified family fails closed if its class-appropriate +executable is absent. The direct completion CLI is intentional: the pinned +`llama-server` completion endpoint does not produce the correct FLAN-T5 text +for this fixture, while the pinned monolithic `llama-completion` encoder pass +and decoder loop do. + +The per-class oracle gates are: + +| Class | Independent comparison | Important limit | +|---|---|---| +| Embedding | Identical batch and each individual input; same vector width, maximum coordinate error `1e-4`, minimum cosine `0.99999` | Parity does not measure retrieval quality | +| Rerank | Identical query/documents; maximum score error `1e-4` and identical ordering | Parity does not measure ranking quality | +| Encoder-decoder | Same prompt and greedy seed; identical text after whitespace normalization | Pinned direct monolithic completion CLI, not the server endpoint | +| OCR | Same generated `MESH 42` PNG and prompt; normalized text matches and contains the independently known fixture label | One synthetic image does not certify broad OCR accuracy | +| Speech recognition | Same WAV and prompt; normalized text matches | The generic smoke WAV has no checked-in transcript label, so this is execution parity, not transcription accuracy | +| Speech synthesis | Same prompt, seed, top-k/top-p, and frame cap in deterministic in-process Skippy and monolithic `llama-tts`; PCM format/length match, relative RMS error at most `2%`, waveform cosine at least `0.9995` | Public HTTP speech sampling is stochastic; PCM parity does not establish intelligibility | + +The family battery records an oracle pass only when the class-specific +comparison completes and a run-local evidence file matches the pinned model, +projector, candidate executable, oracle executable, and patch SHA. A smoke +pass cannot substitute for that evidence. The original provisional +`workload-smoke` profile remains available for future rows that have not yet +passed an independent oracle. Certification here is a bounded equivalence +claim for these six pinned artifacts and fixtures, not broad model-quality +or split-serving certification. diff --git a/docs/README.md b/docs/README.md index 238750fe06..a50f1c9387 100644 --- a/docs/README.md +++ b/docs/README.md @@ -14,6 +14,7 @@ Use this hub to find project guides that are not owned by a single Rust crate. | SDK usage, examples, errors, lifecycle, platform support | [SDK.md](SDK.md) | | Language-specific SDK examples | [Rust](sdk/rust.md), [Node.js](sdk/node.md), [Swift](sdk/swift.md), [Kotlin/Android](sdk/kotlin.md) | | Run big models with Skippy layer splits | [SKIPPY_SPLITS.md](SKIPPY_SPLITS.md) | +| Embedding, rerank, encoder-decoder, OCR, and audio models | [NON_CHAT_MODELS.md](NON_CHAT_MODELS.md) | | Contribute or publish layer package repositories | [LAYER_PACKAGE_REPOS.md](LAYER_PACKAGE_REPOS.md) | | Goose, Claude Code, OpenCode, Pi, curl, blackboard | [AGENTS.md](AGENTS.md) | | Command-by-command CLI reference | [CLI.md](CLI.md) | diff --git a/docs/SKIPPY.md b/docs/SKIPPY.md index b5af69e0ef..9b78559087 100644 --- a/docs/SKIPPY.md +++ b/docs/SKIPPY.md @@ -754,14 +754,21 @@ router it provides. Mesh's public ingress should be able to: - stream responses through the shared adapters without buffering the full response body. -Embeddings can be deferred for now. The replacement plan does not need -`/v1/embeddings` parity before removing `llama-server`. +The shared frontend and local full-model runtime now cover embedding, rerank, +encoder-decoder, OCR, speech-synthesis, and speech-recognition workloads. These +paths are intentionally separate from staged causal generation: unsupported +stage shapes return structured errors instead of inheriting a generation lane. +See [NON_CHAT_MODELS.md](NON_CHAT_MODELS.md) for the current endpoint and +certification matrix. Current branch status: - `openai-frontend` owns `/v1/chat/completions`, `/v1/completions`, and `/v1/responses` request/response shapes, streaming SSE adapters, OpenAI error bodies, tool-call fields, structured-output fields, and logprob fields; +- `openai-frontend` also owns `/v1/embeddings`, `/v1/rerank`, and `/v1/audio/*` + request/response shapes, including bounded multipart uploads and binary audio + responses; - frontend fixture coverage accepts and translates tool, structured-output, logprob, streaming, and responses requests without mesh carrying a second public OpenAI model; diff --git a/docs/design/MULTI_MODAL.md b/docs/design/MULTI_MODAL.md index 67a7e695b0..2a4adade12 100644 --- a/docs/design/MULTI_MODAL.md +++ b/docs/design/MULTI_MODAL.md @@ -12,7 +12,10 @@ Phases 1 through 5 in this plan are complete: - multimodal `/v1/chat/completions` - multimodal `/v1/responses` for both non-streaming and streaming requests -What remains is polish and hardening of the existing path, not new endpoint scope. +The original chat/Responses phases are complete. Full-model audio endpoints +and OCR family certification are documented separately in +[NON_CHAT_MODELS.md](../NON_CHAT_MODELS.md); projector execution remains +colocated with its model trunk. ## Goals @@ -26,10 +29,8 @@ What remains is polish and hardening of the existing path, not new endpoint scop - Permanent distributed file storage - IPFS/libp2p-first design -- `POST /v1/audio/transcriptions` -- `POST /v1/audio/speech` - `v1/realtime` -- Audio generation from llama alone +- Distributed projector execution - Native end-to-end video inference on the current llama.cpp path ## API Targets @@ -47,6 +48,15 @@ This is the shortest path because llama.cpp already supports multimodal chat her Implement as a mesh-llm compatibility shim after chat completions are solid. +### Full-model audio extensions + +- `POST /v1/audio/speech` +- `POST /v1/audio/transcriptions` +- `POST /v1/audio/translations` + +These routes are available for runtime-compatible local model/projector pairs. +They do not imply a split projector or distributed audio-generation graph. + ## Capability Model Do not collapse everything into `vision`. diff --git a/just/ci.just b/just/ci.just index fe36f40e0f..3a58407719 100644 --- a/just/ci.just +++ b/just/ci.just @@ -109,8 +109,13 @@ test-all: --exclude mesh-llm-nodejs \ --exclude skippy-ffi \ --exclude skippy-quantize \ + --exclude skippy-model-package \ --exclude skippy-runtime \ --exclude skippy-server + # The workspace's dynamic-native-runtime feature otherwise unifies into + # the package fixture tests, which require the statically linked GGUF ABI. + echo "--- Static Skippy model-package tests ---" + just with-lld cargo test --package skippy-model-package --bin skippy-model-package echo "--- Static Skippy runtime tests ---" just with-lld cargo test --package skippy-runtime --no-default-features --lib echo "--- Static Skippy server tests ---" diff --git a/scripts/build-llama.sh b/scripts/build-llama.sh index af71f5bfaa..ac4630b43f 100755 --- a/scripts/build-llama.sh +++ b/scripts/build-llama.sh @@ -13,6 +13,7 @@ LLAMA_LINK_MODE="${LLAMA_STAGE_LINK_MODE:-${SKIPPY_LLAMA_LINK_MODE:-static}}" LLAMA_STAGE_BUILD_TESTS="${LLAMA_STAGE_BUILD_TESTS:-OFF}" LLAMA_STAGE_FULL_REPLAY="${LLAMA_STAGE_FULL_REPLAY:-OFF}" LLAMA_STAGE_UPSTREAM_TESTS="${LLAMA_STAGE_UPSTREAM_TESTS:-OFF}" +LLAMA_STAGE_WORKLOAD_ORACLE="${LLAMA_STAGE_WORKLOAD_ORACLE:-OFF}" LLAMA_BUILD_TESTS=OFF LLAMA_BUILD_SERVER=OFF if [[ "$LLAMA_STAGE_FULL_REPLAY" == "ON" || "$LLAMA_STAGE_UPSTREAM_TESTS" == "ON" ]]; then @@ -24,6 +25,9 @@ if [[ "$LLAMA_STAGE_UPSTREAM_TESTS" == "ON" ]]; then # multimodal include/link closure. LLAMA_BUILD_SERVER=ON fi +if [[ "$LLAMA_STAGE_WORKLOAD_ORACLE" == "ON" ]]; then + LLAMA_BUILD_SERVER=ON +fi PRINT_BUILD_DIR=0 REQUIRE_EXISTING=0 @@ -151,9 +155,15 @@ required_dynamic_libraries_exist() { required_outputs_exist() { if [[ "$LLAMA_LINK_MODE" == "dynamic" ]]; then - required_dynamic_libraries_exist + required_dynamic_libraries_exist || return 1 else - required_static_archives_exist + required_static_archives_exist || return 1 + fi + if [[ "$LLAMA_STAGE_WORKLOAD_ORACLE" == "ON" ]]; then + [[ -x "$LLAMA_BUILD_DIR/bin/llama-server" && + -x "$LLAMA_BUILD_DIR/bin/llama-cli" && + -x "$LLAMA_BUILD_DIR/bin/llama-completion" && + -x "$LLAMA_BUILD_DIR/bin/llama-tts" ]] || return 1 fi } @@ -220,6 +230,14 @@ CMAKE_ARGS=( # platforms. -DMTMD_VIDEO=OFF ) +if [[ "$LLAMA_STAGE_WORKLOAD_ORACLE" == "ON" ]]; then + CMAKE_ARGS+=(-DLLAMA_BUILD_COMMON=ON -DLLAMA_BUILD_TOOLS=ON) +fi +if [[ "$LLAMA_BACKEND" == "cpu" ]]; then + # macOS defaults Metal to ON even when the selected backend is CPU. Match + # the backend contract for both the embedded runtime and its test oracle. + CMAKE_ARGS+=(-DGGML_METAL=OFF) +fi # Static ABI inputs cross job and runner boundaries. Normalize compiler- # embedded source/build paths so the archived link closure does not retain a @@ -379,6 +397,10 @@ fi cmake "${CMAKE_ARGS[@]}" BUILD_TARGETS=(llama llama-common mtmd) +if [[ "$LLAMA_STAGE_WORKLOAD_ORACLE" == "ON" ]]; then + # Test-only full-model references. They are never packaged beside the host. + BUILD_TARGETS+=(llama-server llama-cli llama-completion llama-tts) +fi if [[ "$LLAMA_STAGE_BUILD_TESTS" == "ON" ]]; then BUILD_TARGETS+=( skippy-graph-build-inputs @@ -397,6 +419,8 @@ if [[ "$LLAMA_STAGE_FULL_REPLAY" == "ON" ]]; then test-skippy-kv-page-export test-skippy-model-loader-accounting test-skippy-recurrent-state-roundtrip + test-skippy-rerank-template + test-skippy-sampling-suppress test-skippy-verify-checkpoint-retirement ) fi diff --git a/scripts/check-skippy-workload-candidate.py b/scripts/check-skippy-workload-candidate.py new file mode 100644 index 0000000000..4643125efb --- /dev/null +++ b/scripts/check-skippy-workload-candidate.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""Reject a stale statically linked candidate before an oracle comparison.""" + +from __future__ import annotations + +import argparse +import os +from pathlib import Path + + +def check_candidate(binary: Path, build_dir: Path) -> None: + stamp = build_dir / ".mesh-llm-build-stamp" + if not binary.is_file() or not os.access(binary, os.X_OK): + raise RuntimeError(f"candidate executable is missing: {binary}") + if not stamp.is_file(): + raise RuntimeError(f"candidate native build stamp is missing: {stamp}") + if binary.stat().st_mtime_ns <= stamp.stat().st_mtime_ns: + raise RuntimeError( + "candidate executable predates the stamped native ABI; " + "rebuild skippy-server against the current LLAMA_STAGE_BUILD_DIR" + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidate-binary", required=True, type=Path) + parser.add_argument("--native-build-dir", required=True, type=Path) + args = parser.parse_args() + check_candidate(args.candidate_binary, args.native_build_dir) + + +if __name__ == "__main__": + main() diff --git a/scripts/ci-openai-embeddings-smoke.py b/scripts/ci-openai-embeddings-smoke.py new file mode 100755 index 0000000000..2ad1f8c6fb --- /dev/null +++ b/scripts/ci-openai-embeddings-smoke.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Official openai-python embeddings smoke against a compatible endpoint.""" + +from __future__ import annotations + +import argparse +import base64 +import math +import struct + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", required=True) + parser.add_argument("--model", required=True) + args = parser.parse_args() + + try: + from openai import OpenAI + except ModuleNotFoundError as exc: + raise SystemExit( + "openai package not installed; run `python -m pip install openai` first" + ) from exc + + client = OpenAI(api_key="mesh-llm-ci", base_url=args.base_url) + inputs = [ + "search_query: distributed inference", + "search_document: GPUs collaborate over a mesh", + ] + response = client.embeddings.create( + model=args.model, + input=inputs, + encoding_format="float", + ) + if response.object != "list" or response.model != args.model: + raise RuntimeError("embeddings response has the wrong object or model") + if len(response.data) != len(inputs): + raise RuntimeError("embeddings response has the wrong batch size") + dimensions: int | None = None + for index, item in enumerate(response.data): + if item.object != "embedding" or item.index != index: + raise RuntimeError("embeddings response has invalid item metadata") + if not item.embedding or not all(math.isfinite(value) for value in item.embedding): + raise RuntimeError("embeddings response contains no finite vector") + norm = math.sqrt(sum(value * value for value in item.embedding)) + if abs(norm - 1.0) > 1e-4: + raise RuntimeError(f"embedding {index} is not L2-normalized: {norm}") + if dimensions is None: + dimensions = len(item.embedding) + elif dimensions != len(item.embedding): + raise RuntimeError("embedding dimensions differ within one response") + if response.usage.prompt_tokens <= 0: + raise RuntimeError("embeddings usage reports no prompt tokens") + + encoded = client.embeddings.create( + model=args.model, + input=inputs[0], + encoding_format="base64", + ) + payload = encoded.data[0].embedding + if not isinstance(payload, str): + raise RuntimeError("base64 embedding did not deserialize as a string") + raw = base64.b64decode(payload, validate=True) + if dimensions is None or len(raw) != dimensions * struct.calcsize(" dict: + request = urllib.request.Request( + f"{base_url}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=240) as response: + if response.headers.get_content_type() != "application/json": + raise RuntimeError(f"{path} returned non-JSON content") + result = json.load(response) + if not isinstance(result, dict): + raise RuntimeError(f"{path} returned a non-object response") + return result + + +def request_bytes(base_url: str, path: str, payload: dict[str, object]) -> tuple[str, bytes]: + request = urllib.request.Request( + f"{base_url}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=240) as response: + return response.headers.get_content_type(), response.read() + + +def request_multipart(base_url: str, path: str, model: str, media_path: Path) -> dict: + boundary = "mesh-llm-workload-smoke" + body = ( + f"--{boundary}\r\n" + 'Content-Disposition: form-data; name="model"\r\n\r\n' + f"{model}\r\n" + f"--{boundary}\r\n" + f'Content-Disposition: form-data; name="file"; filename="{media_path.name}"\r\n' + "Content-Type: audio/wav\r\n\r\n" + ).encode("utf-8") + body += media_path.read_bytes() + f"\r\n--{boundary}--\r\n".encode("ascii") + request = urllib.request.Request( + f"{base_url}{path}", + data=body, + headers={"Content-Type": f"multipart/form-data; boundary={boundary}"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=240) as response: + if response.headers.get_content_type() != "application/json": + raise RuntimeError(f"{path} returned non-JSON content") + result = json.load(response) + if not isinstance(result, dict): + raise RuntimeError(f"{path} returned a non-object response") + return result + + +def smoke_embedding(base_url: str, model: str) -> None: + inputs = list(EMBEDDING_INPUTS) + result = request_json( + base_url, + "/embeddings", + {"model": model, "input": inputs, "encoding_format": "float"}, + ) + rows = result.get("data") + if result.get("object") != "list" or result.get("model") != model: + raise RuntimeError("embedding response has the wrong object or model") + if not isinstance(rows, list) or len(rows) != len(inputs): + raise RuntimeError("embedding response has the wrong batch size") + vectors = [] + for index, row in enumerate(rows): + values = row.get("embedding") + if row.get("object") != "embedding" or row.get("index") != index: + raise RuntimeError("embedding response has invalid item metadata") + if not isinstance(values, list) or not values: + raise RuntimeError("embedding response has no vector") + if not all(isinstance(value, (int, float)) and math.isfinite(value) for value in values): + raise RuntimeError("embedding response contains non-finite values") + norm = math.sqrt(sum(value * value for value in values)) + if abs(norm - 1.0) > 1e-4: + raise RuntimeError(f"embedding {index} is not normalized: {norm}") + vectors.append(values) + if len({len(vector) for vector in vectors}) != 1: + raise RuntimeError("embedding dimensions differ within one response") + related = sum(left * right for left, right in zip(vectors[0], vectors[1])) + unrelated = sum(left * right for left, right in zip(vectors[0], vectors[2])) + if related <= unrelated: + raise RuntimeError( + f"embedding placed unrelated text closer: related={related}, unrelated={unrelated}" + ) + if result.get("usage", {}).get("prompt_tokens", 0) <= 0: + raise RuntimeError("embedding response reported no prompt usage") + encoded = request_json( + base_url, + "/embeddings", + {"model": model, "input": inputs[0], "encoding_format": "base64"}, + ) + payload = encoded.get("data", [{}])[0].get("embedding") + if not isinstance(payload, str): + raise RuntimeError("base64 embedding is not a string") + raw = base64.b64decode(payload, validate=True) + if len(raw) != len(vectors[0]) * struct.calcsize(" None: + result = request_json( + base_url, + "/rerank", + { + "model": model, + "query": RERANK_QUERY, + "documents": list(RERANK_DOCUMENTS), + "return_documents": True, + }, + ) + rows = result.get("results") + if not isinstance(rows, list) or len(rows) != 2: + raise RuntimeError("rerank did not return both documents") + if sorted(row.get("index") for row in rows) != [0, 1]: + raise RuntimeError("rerank returned invalid document indexes") + for row in rows: + score = row.get("relevance_score") + if not isinstance(score, (int, float)) or not math.isfinite(score): + raise RuntimeError("rerank returned a non-finite score") + if not isinstance(row.get("document"), str): + raise RuntimeError("rerank omitted the requested document") + scores = {row["index"]: row["relevance_score"] for row in rows} + if scores[0] <= scores[1]: + raise RuntimeError(f"rerank misplaced the relevant document: {scores}") + if result.get("usage", {}).get("prompt_tokens", 0) <= 0: + raise RuntimeError("rerank did not report prompt usage") + + +def smoke_encoder_decoder(base_url: str, model: str) -> None: + result = request_json( + base_url, + "/completions", + { + "model": model, + "prompt": ENCODER_DECODER_PROMPT, + "max_tokens": 32, + "temperature": 0.0, + }, + ) + choices = result.get("choices") + if not isinstance(choices, list) or len(choices) != 1: + raise RuntimeError("encoder-decoder completion has invalid choices") + if not isinstance(choices[0].get("text"), str) or not choices[0]["text"].strip(): + raise RuntimeError("encoder-decoder completion is empty") + if "haus" not in choices[0]["text"].casefold(): + raise RuntimeError(f"encoder-decoder missed the translation anchor: {choices[0]['text']!r}") + if result.get("usage", {}).get("completion_tokens", 0) <= 0: + raise RuntimeError("encoder-decoder completion reported no generated tokens") + + +def smoke_ocr(base_url: str, model: str, media_path: Path) -> None: + image = base64.b64encode(media_path.read_bytes()).decode("ascii") + result = request_json( + base_url, + "/chat/completions", + { + "model": model, + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Read all visible text. Return only the transcription."}, + {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{image}"}}, + ], + } + ], + "max_tokens": 32, + "temperature": 0.0, + }, + ) + choices = result.get("choices") + if not isinstance(choices, list) or len(choices) != 1: + raise RuntimeError("OCR chat completion has invalid choices") + text = choices[0].get("message", {}).get("content") + if not isinstance(text, str) or not text.strip(): + raise RuntimeError("OCR chat completion is empty") + + +def smoke_speech_synthesis(base_url: str, model: str) -> None: + content_type, audio = request_bytes( + base_url, + "/audio/speech", + { + "model": model, + "input": "The mesh is ready.", + "voice": "default", + "response_format": "wav", + }, + ) + if content_type != "audio/wav": + raise RuntimeError(f"speech synthesis returned {content_type}, not audio/wav") + with wave.open(io.BytesIO(audio)) as sound: + if sound.getnframes() < sound.getframerate() // 10: + raise RuntimeError("speech synthesis returned too little audio") + if sound.getnchannels() < 1 or sound.getsampwidth() != 2: + raise RuntimeError("speech synthesis returned unsupported WAV samples") + samples = sound.readframes(sound.getnframes()) + values = struct.unpack(f"<{len(samples) // 2}h", samples) + rms = math.sqrt(sum(value * value for value in values) / len(values)) + if rms < 1.0: + raise RuntimeError("speech synthesis returned silent audio") + + +def smoke_speech_recognition(base_url: str, model: str, media_path: Path) -> None: + result = request_multipart(base_url, "/audio/transcriptions", model, media_path) + text = result.get("text") + if not isinstance(text, str) or not text.strip(): + raise RuntimeError("speech recognition returned no transcription") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", required=True) + parser.add_argument("--model", required=True) + parser.add_argument( + "--class", + dest="model_class", + required=True, + choices=("embedding", "rerank", "encoder_decoder", "ocr", "speech_synthesis", "speech_recognition"), + ) + parser.add_argument("--media-path", type=Path) + args = parser.parse_args() + + if args.model_class in {"ocr", "speech_recognition"}: + if args.media_path is None or not args.media_path.is_file(): + parser.error(f"{args.model_class} requires --media-path pointing to a file") + checks = { + "embedding": lambda: smoke_embedding(args.base_url, args.model), + "rerank": lambda: smoke_rerank(args.base_url, args.model), + "encoder_decoder": lambda: smoke_encoder_decoder(args.base_url, args.model), + "ocr": lambda: smoke_ocr(args.base_url, args.model, args.media_path), + "speech_synthesis": lambda: smoke_speech_synthesis(args.base_url, args.model), + "speech_recognition": lambda: smoke_speech_recognition(args.base_url, args.model, args.media_path), + } + checks[args.model_class]() + print(f"OpenAI HTTP {args.model_class} smoke passed: model={args.model}") + + +if __name__ == "__main__": + main() diff --git a/scripts/ci-workload-monolithic-oracle.py b/scripts/ci-workload-monolithic-oracle.py new file mode 100644 index 0000000000..59b8eb697e --- /dev/null +++ b/scripts/ci-workload-monolithic-oracle.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 +"""Compare one Skippy non-chat response with pinned llama.cpp full-model serving. + +Only classes whose OpenAI request and result can be aligned with llama-server +are supported here. This is a numerical/output parity gate, not a general +model-quality benchmark. Both servers must load the same immutable GGUF on CPU. +""" + +from __future__ import annotations + +import argparse +import json +import math +import re +import subprocess +import urllib.request + +from workload_fixtures import ( + EMBEDDING_INPUTS, + ENCODER_DECODER_PROMPT, + RERANK_DOCUMENTS, + RERANK_QUERY, +) + + +EMBEDDING_MAX_ABS_DELTA = 1e-4 +EMBEDDING_MIN_COSINE = 0.99999 +RERANK_MAX_ABS_DELTA = 1e-4 + + +def request_json(base_url: str, path: str, payload: dict[str, object]) -> dict: + request = urllib.request.Request( + f"{base_url}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=240) as response: + if response.headers.get_content_type() != "application/json": + raise RuntimeError(f"{base_url}{path} returned non-JSON content") + result = json.load(response) + if not isinstance(result, dict): + raise RuntimeError(f"{base_url}{path} returned a non-object response") + return result + + +def vectors(response: dict, expected_count: int) -> list[list[float]]: + rows = response.get("data") + if not isinstance(rows, list) or len(rows) != expected_count: + raise RuntimeError("embedding oracle response has the wrong batch size") + result = [] + for index, row in enumerate(rows): + if not isinstance(row, dict) or row.get("index") != index: + raise RuntimeError("embedding oracle response has invalid indexes") + vector = row.get("embedding") + if not isinstance(vector, list) or not vector: + raise RuntimeError("embedding oracle response has no vector") + if not all(type(value) in (int, float) and math.isfinite(value) for value in vector): + raise RuntimeError("embedding oracle response has non-finite values") + result.append(vector) + return result + + +def compare_embeddings(candidate: dict, reference: dict, expected_count: int = len(EMBEDDING_INPUTS)) -> str: + candidate_vectors = vectors(candidate, expected_count) + reference_vectors = vectors(reference, expected_count) + max_delta = 0.0 + min_cosine = 1.0 + row_metrics = [] + for index, (candidate_vector, reference_vector) in enumerate(zip( + candidate_vectors, reference_vectors, strict=True + )): + if len(candidate_vector) != len(reference_vector): + raise RuntimeError("embedding dimensions differ from monolithic reference") + row_delta = max(abs(left - right) for left, right in zip(candidate_vector, reference_vector)) + max_delta = max(max_delta, row_delta) + dot = sum(left * right for left, right in zip(candidate_vector, reference_vector)) + left_norm = math.sqrt(sum(value * value for value in candidate_vector)) + right_norm = math.sqrt(sum(value * value for value in reference_vector)) + if left_norm == 0 or right_norm == 0: + raise RuntimeError("embedding oracle response has a zero vector") + cosine = dot / (left_norm * right_norm) + min_cosine = min(min_cosine, cosine) + row_metrics.append(f"{index}:delta={row_delta:.7g},cos={cosine:.8g}") + if max_delta > EMBEDDING_MAX_ABS_DELTA or min_cosine < EMBEDDING_MIN_COSINE: + raise RuntimeError( + "embedding differs from monolithic reference: " + f"max_abs_delta={max_delta:.7g}, min_cosine={min_cosine:.8g}, " + f"rows=[{'; '.join(row_metrics)}], " + f"candidate_head={candidate_vectors[0][:5]}, reference_head={reference_vectors[0][:5]}, " + f"candidate_usage={candidate.get('usage')}, reference_usage={reference.get('usage')}" + ) + return f"max_abs_delta={max_delta:.7g}, min_cosine={min_cosine:.8g}" + + +def run_embedding_oracle(candidate_url: str, oracle_url: str, model: str) -> str: + payload = {"model": model, "input": list(EMBEDDING_INPUTS), "encoding_format": "float"} + candidate = request_json(candidate_url, "/embeddings", payload) + reference = request_json(oracle_url, "/embeddings", payload) + failures = [] + try: + batch_detail = compare_embeddings(candidate, reference) + except RuntimeError as error: + failures.append(f"batched: {error}") + batch_detail = "failed" + single_details = [] + for index, text in enumerate(EMBEDDING_INPUTS): + single_payload = {"model": model, "input": text, "encoding_format": "float"} + candidate = request_json(candidate_url, "/embeddings", single_payload) + reference = request_json(oracle_url, "/embeddings", single_payload) + try: + single_details.append(compare_embeddings(candidate, reference, expected_count=1)) + except RuntimeError as error: + failures.append(f"single[{index}]: {error}") + if failures: + raise RuntimeError("; ".join(failures)) + return f"batch {batch_detail}; singles {', '.join(single_details)}" + + +def indexed_scores(response: dict) -> dict[int, float]: + rows = response.get("results") + if not isinstance(rows, list) or len(rows) != len(RERANK_DOCUMENTS): + raise RuntimeError("rerank oracle response has the wrong document count") + scores = {} + for row in rows: + if not isinstance(row, dict): + raise RuntimeError("rerank oracle response has an invalid row") + index = row.get("index") + score = row.get("relevance_score") + if type(index) is not int or index not in range(len(RERANK_DOCUMENTS)): + raise RuntimeError("rerank oracle response has an invalid document index") + if index in scores or type(score) not in (int, float) or not math.isfinite(score): + raise RuntimeError("rerank oracle response has a duplicate or non-finite score") + scores[index] = float(score) + return scores + + +def compare_rerank(candidate: dict, reference: dict) -> str: + candidate_scores = indexed_scores(candidate) + reference_scores = indexed_scores(reference) + max_delta = max( + abs(candidate_scores[index] - reference_scores[index]) + for index in range(len(RERANK_DOCUMENTS)) + ) + candidate_order = sorted(candidate_scores, key=lambda index: -candidate_scores[index]) + reference_order = sorted(reference_scores, key=lambda index: -reference_scores[index]) + if candidate_order != reference_order or max_delta > RERANK_MAX_ABS_DELTA: + raise RuntimeError( + "rerank differs from monolithic reference: " + f"candidate_order={candidate_order}, reference_order={reference_order}, " + f"max_abs_delta={max_delta:.7g}" + ) + return f"max_abs_delta={max_delta:.7g}, order={candidate_order}" + + +def completion_text(response: dict) -> str: + choices = response.get("choices") + if not isinstance(choices, list) or len(choices) != 1: + raise RuntimeError("encoder-decoder oracle response has invalid choices") + text = choices[0].get("text") if isinstance(choices[0], dict) else None + if not isinstance(text, str) or not text.strip(): + raise RuntimeError("encoder-decoder oracle response has no text") + return re.sub(r"\s+", " ", text).strip() + + +def compare_encoder_decoder(candidate: dict, reference: dict) -> str: + candidate_text = completion_text(candidate) + reference_text = completion_text(reference) + if candidate_text != reference_text: + raise RuntimeError( + "encoder-decoder text differs from monolithic reference: " + f"candidate={candidate_text!r}, reference={reference_text!r}" + ) + return f"identical normalized text={candidate_text!r}" + + +def monolithic_completion(oracle_cli: str, model_path: str) -> dict: + command = [ + oracle_cli, "-m", model_path, "-p", ENCODER_DECODER_PROMPT, + "-n", "32", "-c", "0", "-b", "2048", "-ub", "2048", "-ngl", "0", + "-s", "1", "--temp", "0", "--no-repack", + "--no-display-prompt", "--simple-io", + ] + try: + result = subprocess.run(command, capture_output=True, text=True, timeout=240, check=False) + except subprocess.TimeoutExpired as error: + raise RuntimeError("monolithic encoder-decoder completion timed out") from error + if result.returncode != 0: + raise RuntimeError( + f"monolithic encoder-decoder completion exited {result.returncode}: " + f"{result.stderr[-1000:]}" + ) + # llama-completion prints this terminal marker after the model emits EOG; + # it is runner metadata, not generated model text. + text = re.sub(r"\s*\[end of text\]\s*$", "", result.stdout).strip() + if not text: + raise RuntimeError("monolithic encoder-decoder completion produced no text") + return {"choices": [{"text": text}]} + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidate-url", required=True) + parser.add_argument("--oracle-url") + parser.add_argument("--oracle-completion") + parser.add_argument("--model-path") + parser.add_argument("--model", required=True) + parser.add_argument("--class", dest="model_class", required=True, + choices=("embedding", "rerank", "encoder_decoder")) + args = parser.parse_args() + + if args.model_class == "encoder_decoder": + if not args.oracle_completion or not args.model_path or args.oracle_url: + parser.error("encoder_decoder requires --oracle-completion and --model-path only") + payload = {"model": args.model, "prompt": ENCODER_DECODER_PROMPT, + "max_tokens": 32, "temperature": 0.0, "seed": 1} + candidate = request_json(args.candidate_url, "/completions", payload) + reference = monolithic_completion(args.oracle_completion, args.model_path) + detail = compare_encoder_decoder(candidate, reference) + print(f"encoder_decoder local-monolithic oracle passed: {detail}") + return + if not args.oracle_url or args.oracle_completion or args.model_path: + parser.error("embedding/rerank require --oracle-url only") + if args.model_class == "embedding": + detail = run_embedding_oracle(args.candidate_url, args.oracle_url, args.model) + print(f"embedding local-monolithic oracle passed: {detail}") + return + + requests = { + "rerank": ( + "/rerank", + {"model": args.model, "query": RERANK_QUERY, + "documents": list(RERANK_DOCUMENTS), "return_documents": True}, + compare_rerank, + ), + } + path, payload, comparator = requests[args.model_class] + candidate = request_json(args.candidate_url, path, payload) + reference = request_json(args.oracle_url, path, payload) + detail = comparator(candidate, reference) + print(f"{args.model_class} local-monolithic oracle passed: {detail}") + + +if __name__ == "__main__": + main() diff --git a/scripts/generate-ocr-oracle-fixture.py b/scripts/generate-ocr-oracle-fixture.py new file mode 100644 index 0000000000..e4e27ba5cb --- /dev/null +++ b/scripts/generate-ocr-oracle-fixture.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Generate an original, deterministic text-bearing PNG for the OCR oracle. + +The 5x7 glyphs below were drawn for this repository. No fonts, image assets, +network services, or third-party packages are required. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path +import struct +import zlib + + +TEXT = "MESH 42" +GLYPHS = { + "M": ("10001", "11011", "10101", "10101", "10001", "10001", "10001"), + "E": ("11111", "10000", "10000", "11110", "10000", "10000", "11111"), + "S": ("01111", "10000", "10000", "01110", "00001", "00001", "11110"), + "H": ("10001", "10001", "10001", "11111", "10001", "10001", "10001"), + "4": ("00110", "01010", "10010", "10010", "11111", "00010", "00010"), + "2": ("11110", "00001", "00001", "01110", "10000", "10000", "11111"), + " ": ("00000",) * 7, +} +SCALE = 12 +MARGIN = 36 +CHAR_SPACING = SCALE +WIDTH = 2 * MARGIN + len(TEXT) * (5 * SCALE + CHAR_SPACING) - CHAR_SPACING +HEIGHT = 2 * MARGIN + 7 * SCALE + + +def chunk(kind: bytes, data: bytes) -> bytes: + payload = kind + data + return struct.pack(">I", len(data)) + payload + struct.pack(">I", zlib.crc32(payload)) + + +def png_bytes() -> bytes: + pixels = bytearray(b"\xff" * (WIDTH * HEIGHT * 3)) + for char_index, char in enumerate(TEXT): + for glyph_y, row in enumerate(GLYPHS[char]): + for glyph_x, bit in enumerate(row): + if bit == "0": + continue + x0 = MARGIN + char_index * (5 * SCALE + CHAR_SPACING) + glyph_x * SCALE + y0 = MARGIN + glyph_y * SCALE + for y in range(y0, y0 + SCALE): + for x in range(x0, x0 + SCALE): + offset = 3 * (y * WIDTH + x) + pixels[offset : offset + 3] = b"\x00\x00\x00" + rows = b"".join( + b"\x00" + pixels[y * WIDTH * 3 : (y + 1) * WIDTH * 3] + for y in range(HEIGHT) + ) + return ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", struct.pack(">IIBBBBB", WIDTH, HEIGHT, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(rows, level=9)) + + chunk(b"IEND", b"") + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + args.output.write_bytes(png_bytes()) + print(f"wrote {TEXT!r} OCR oracle fixture: {args.output}") + + +if __name__ == "__main__": + main() diff --git a/scripts/generate-test-model-manifests.py b/scripts/generate-test-model-manifests.py index b69fc360b1..d0d76bb624 100644 --- a/scripts/generate-test-model-manifests.py +++ b/scripts/generate-test-model-manifests.py @@ -25,6 +25,15 @@ SHA_RE = re.compile(r"^[0-9a-f]{40,64}$") HASH_RE = re.compile(r"^[0-9a-f]{64}$") ID_RE = re.compile(r"^[a-z0-9][a-z0-9._-]*$") +FAMILY_WORKLOAD_CLASSES = { + "causal_generation", + "embedding", + "rerank", + "encoder_decoder", + "ocr", + "speech_synthesis", + "speech_recognition", +} SUITE_OUTPUTS = { "product-smoke": MANIFEST_DIR / "product-smoke.json", @@ -130,9 +139,9 @@ def _validate_registry(raw: Any) -> dict[str, Any]: if policy.get("cadences") != ["llama-bump", "manual-full", "nightly", "rotating"]: raise RegistryError("registry.family_policy.cadences must preserve family cadence order") profiles = _object(policy.get("profiles"), "registry.family_policy.profiles") - expected_profiles = {"full", "package-oracle", "graph-only"} + expected_profiles = {"full", "package-oracle", "graph-only", "workload-smoke", "workload-oracle"} if set(profiles) != expected_profiles: - raise RegistryError("registry.family_policy.profiles must contain the three family profiles") + raise RegistryError("registry.family_policy.profiles must contain the five family profiles") for profile_name, profile in profiles.items(): profile = _object(profile, f"registry.family_policy.profiles.{profile_name}") _exact_keys(profile, {"status", "oracle", "required_lanes"}, f"profile {profile_name}") @@ -175,12 +184,24 @@ def _validate_registry(raw: Any) -> dict[str, Any]: certification = _object(row.get("certification"), f"{field}.certification") _exact_keys( certification, - {"profile", "cadences", "execution", "resources", "notes", "draft_artifact", "mmproj_artifact"}, + {"class", "profile", "cadences", "execution", "resources", "notes", "evidence", "draft_artifact", "mmproj_artifact"}, f"{field}.certification", ) + workload_class = _string( + certification.get("class"), f"{field}.certification.class" + ) + if workload_class not in FAMILY_WORKLOAD_CLASSES: + raise RegistryError(f"{field}.certification.class is not a workload class") profile = _string(certification.get("profile"), f"{field}.certification.profile") if profile not in profiles: raise RegistryError(f"{field}.certification.profile is not a family profile") + if profile == "workload-oracle": + evidence = _object(certification.get("evidence"), f"{field}.certification.evidence") + _exact_keys(evidence, {"fixture", "comparison"}, f"{field}.certification.evidence") + _string(evidence.get("fixture"), f"{field}.certification.evidence.fixture") + _string(evidence.get("comparison"), f"{field}.certification.evidence.comparison") + elif "evidence" in certification: + raise RegistryError(f"{field}.certification.evidence requires workload-oracle") if "cadences" in certification: certification_cadences = _string_list( certification["cadences"], f"{field}.certification.cadences" @@ -231,6 +252,7 @@ def _family_manifest(registry: dict[str, Any]) -> dict[str, Any]: certification = row["certification"] model: dict[str, Any] = { "family": row["family"], + "class": certification["class"], "profile": certification["profile"], "cadences": certification.get("cadences", row["cadences"]), "artifact": _family_artifact(row["artifact"]), @@ -238,6 +260,8 @@ def _family_manifest(registry: dict[str, Any]) -> dict[str, Any]: for optional in ("draft_artifact", "mmproj_artifact"): if optional in certification: model[optional] = _family_artifact(certification[optional]) + if "evidence" in certification: + model["evidence"] = certification["evidence"] model.update( execution=certification["execution"], resources=certification["resources"], @@ -322,6 +346,7 @@ def compact(item: Any) -> str: [ " {", f' "family": {compact(model["family"])},', + f' "class": {compact(model["class"])},', f' "profile": {compact(model["profile"])},', f' "cadences": {compact(model["cadences"])},', f' "artifact": {compact(model["artifact"])},', @@ -330,6 +355,8 @@ def compact(item: Any) -> str: for optional in ("draft_artifact", "mmproj_artifact"): if optional in model: lines.append(f' {compact(optional)}: {compact(model[optional])},') + if "evidence" in model: + lines.append(f' "evidence": {compact(model["evidence"])},') lines.extend( [ f' "execution": {compact(model["execution"])},', diff --git a/scripts/llama-oracle-source.py b/scripts/llama-oracle-source.py new file mode 100644 index 0000000000..198af68e85 --- /dev/null +++ b/scripts/llama-oracle-source.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Verify that a test oracle uses the current pinned llama.cpp patch queue.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path +import re +import subprocess +import sys + + +ROOT = Path(__file__).resolve().parents[1] + + +def ordered_patches(patch_dir: Path) -> list[Path]: + patches = sorted(patch_dir.glob("*.patch")) + for expected, patch in enumerate(patches, start=1): + if not re.fullmatch(rf"{expected:04d}-.+\.patch", patch.name): + raise RuntimeError(f"invalid top-level patch sequence: {patch.name}") + generated = patch_dir / "generated" + if not generated.exists(): + return patches + series = generated / "series" + if not series.is_file(): + raise RuntimeError("generated patch directory has no series file") + names = [line.rstrip("\r") for line in series.read_text(encoding="utf-8").splitlines()] + if not names or len(names) != len(list(generated.glob("*.patch"))): + raise RuntimeError("generated patch series does not cover its patch directory") + for expected, name in enumerate(names, start=1): + if not re.fullmatch(rf"{expected:04d}-family-[a-z0-9.-]+(?:--[a-z0-9.-]+)*\.patch", name): + raise RuntimeError(f"invalid generated patch sequence: {name}") + patch = generated / name + if not patch.is_file(): + raise RuntimeError(f"generated patch is missing: {name}") + patches.append(patch) + return patches + + +def patch_digest(patch_dir: Path) -> str: + digest = hashlib.sha256() + for patch in ordered_patches(patch_dir): + relative_name = patch.relative_to(patch_dir).as_posix() + file_digest = hashlib.sha256(patch.read_bytes()).hexdigest() + digest.update(f"{relative_name}\n{file_digest}\n".encode("utf-8")) + return digest.hexdigest() + + +def prepared_patched_sha(root: Path) -> str: + checkout = root / ".deps/llama.cpp" + prepared_upstream = (checkout / ".mesh-llm-upstream-sha").read_text(encoding="utf-8").strip() + prepared_patch_digest = (checkout / ".mesh-llm-patch-digest").read_text(encoding="utf-8").strip() + prepared_patched = (checkout / ".mesh-llm-patched-sha").read_text(encoding="utf-8").strip() + prepared_schema = (checkout / ".mesh-llm-prepare-schema").read_text(encoding="utf-8").strip() + upstream = (root / "third_party/llama.cpp/upstream.txt").read_text(encoding="utf-8").strip() + if prepared_schema != "4" or prepared_upstream != upstream: + raise RuntimeError("prepared llama.cpp checkout does not match the pinned upstream") + if prepared_patch_digest != patch_digest(root / "third_party/llama.cpp/patches"): + raise RuntimeError("prepared llama.cpp checkout does not match the current patch queue") + head = subprocess.run( + ["git", "-C", str(checkout), "rev-parse", "HEAD"], + capture_output=True, text=True, check=True, + ).stdout.strip() + if prepared_patched != head: + raise RuntimeError("prepared llama.cpp patched SHA does not match Git HEAD") + subprocess.run(["git", "-C", str(checkout), "diff-index", "--quiet", "HEAD", "--"], + check=True) + return prepared_patched + + +def main() -> None: + try: + print(prepared_patched_sha(ROOT)) + except (OSError, RuntimeError, subprocess.CalledProcessError) as error: + print(f"pinned llama.cpp oracle source is stale: {error}", file=sys.stderr) + raise SystemExit(1) from error + + +if __name__ == "__main__": + main() diff --git a/scripts/plan-family-battery.py b/scripts/plan-family-battery.py index ec6a1dd355..756837c053 100755 --- a/scripts/plan-family-battery.py +++ b/scripts/plan-family-battery.py @@ -24,7 +24,16 @@ ROOT = Path(__file__).resolve().parents[1] DEFAULT_MANIFEST = ROOT / "ci" / "llama-canary" / "family-certified.json" CORE_LANES = ("single-step", "chain", "state-handoff") -PROFILE_NAMES = ("full", "package-oracle", "graph-only") +MODEL_CLASS_LANES = { + "causal_generation": CORE_LANES, + "embedding": ("embedding-smoke", "embedding-oracle"), + "rerank": ("rerank-smoke", "rerank-oracle"), + "encoder_decoder": ("encoder-decoder-smoke", "encoder-decoder-oracle"), + "ocr": ("ocr-smoke", "ocr-oracle"), + "speech_synthesis": ("speech-synthesis-smoke", "speech-synthesis-oracle"), + "speech_recognition": ("speech-recognition-smoke", "speech-recognition-oracle"), +} +PROFILE_NAMES = ("full", "package-oracle", "graph-only", "workload-smoke", "workload-oracle") CERTIFIED_PROFILES = ("full", "package-oracle") CERTIFICATION_STATUSES = ("certified", "provisional") ORACLE_KINDS = ("local-monolithic", "independent-trace", "none") @@ -281,7 +290,9 @@ def _validate_policy(value: object) -> dict[str, Any]: profiles = _object(policy.get("profiles"), "policy.profiles") if set(profiles) != set(PROFILE_NAMES): - raise PlanError("policy.profiles must define full, package-oracle, and graph-only") + raise PlanError( + "policy.profiles must define full, package-oracle, graph-only, workload-smoke, and workload-oracle" + ) normalized: dict[str, Any] = {} for name in PROFILE_NAMES: profile = _object(profiles[name], f"policy.profiles.{name}") @@ -303,8 +314,19 @@ def _validate_policy(value: object) -> dict[str, Any]: raise PlanError( f"certified profile {name} must require exactly the three core lanes" ) + elif name == "workload-oracle": + if status != "certified" or oracle != "local-monolithic" or tuple(lanes) != ( + "class-specific-smoke", "class-specific-oracle" + ): + raise PlanError("workload-oracle requires certified local-monolithic smoke and oracle lanes") elif status != "provisional" or oracle != "none": - raise PlanError("graph-only must remain provisional and oracle-free") + raise PlanError(f"{name} must remain provisional and oracle-free") + elif name == "graph-only" and tuple(lanes) != ( + "graph-parse", "tensor-ownership", "stage-load" + ): + raise PlanError("graph-only must require exactly the three graph lanes") + elif name == "workload-smoke" and tuple(lanes) != ("class-specific-smoke",): + raise PlanError("workload-smoke must require exactly the class-specific smoke lane") normalized[name] = { "status": status, "oracle": oracle, @@ -329,6 +351,7 @@ def _normalize_models(value: object, policy: dict[str, Any]) -> list[dict[str, A model, { "family", + "class", "profile", "cadences", "artifact", @@ -337,6 +360,7 @@ def _normalize_models(value: object, policy: dict[str, Any]) -> list[dict[str, A "execution", "resources", "notes", + "evidence", }, field, ) @@ -346,7 +370,28 @@ def _normalize_models(value: object, policy: dict[str, Any]) -> list[dict[str, A if family in seen: raise PlanError(f"duplicate family: {family}") seen.add(family) + model_class = _enum( + model.get("class"), + f"{field}.class", + tuple(MODEL_CLASS_LANES), + ) profile = _enum(model.get("profile"), f"{field}.profile", PROFILE_NAMES) + if model_class != "causal_generation" and profile not in ("workload-smoke", "workload-oracle"): + raise PlanError( + f"{field}.class {model_class} requires a class-specific workload profile" + ) + if model_class == "causal_generation" and profile in ("workload-smoke", "workload-oracle"): + raise PlanError(f"{field}.class causal_generation cannot use workload profiles") + evidence = None + if profile == "workload-oracle": + evidence = _object(model.get("evidence"), f"{field}.evidence") + _exact_keys(evidence, {"fixture", "comparison"}, f"{field}.evidence") + evidence = { + "fixture": _string(evidence.get("fixture"), f"{field}.evidence.fixture"), + "comparison": _string(evidence.get("comparison"), f"{field}.evidence.comparison"), + } + elif "evidence" in model: + raise PlanError(f"{field}.evidence requires workload-oracle") cadences = _string_list(model.get("cadences"), f"{field}.cadences") if not cadences or any(item not in policy["cadences"] for item in cadences): raise PlanError(f"{field}.cadences contains an unsupported cadence") @@ -359,6 +404,13 @@ def _normalize_models(value: object, policy: dict[str, Any]) -> list[dict[str, A mmproj = _artifact(model["mmproj_artifact"], f"{field}.mmproj_artifact") if mmproj is not None and len(mmproj["files"]) != 1: raise PlanError(f"{field}.mmproj_artifact.files must name exactly one projector GGUF") + projector_classes = {"ocr", "speech_synthesis", "speech_recognition"} + if model_class in projector_classes and mmproj is None: + raise PlanError(f"{field}.class {model_class} requires an mmproj_artifact") + if model_class != "causal_generation" and len(artifact["files"]) != 1: + raise PlanError( + f"{field}.class {model_class} requires exactly one target GGUF" + ) execution = _object(model.get("execution"), f"{field}.execution") _exact_keys( @@ -395,6 +447,15 @@ def _normalize_models(value: object, policy: dict[str, Any]) -> list[dict[str, A f"{field}.execution.speculative_policy", SPECULATIVE_POLICIES, ) + if model_class != "causal_generation": + if mtp_layers != 0 or sweep_period != 0: + raise PlanError( + f"{field}.class {model_class} must not request split or MTP certification" + ) + if speculative_policy != "disabled": + raise PlanError( + f"{field}.class {model_class} must disable speculative decoding" + ) resources = _object(model.get("resources"), f"{field}.resources") _exact_keys( @@ -435,10 +496,15 @@ def _normalize_models(value: object, policy: dict[str, Any]) -> list[dict[str, A models.append( { "family": family, + "class": model_class, "profile": profile, "certification_status": profile_policy["status"], "oracle": profile_policy["oracle"], - "certification_lanes": profile_policy["required_lanes"], + "certification_lanes": ( + profile_policy["required_lanes"] + if model_class == "causal_generation" + else list(MODEL_CLASS_LANES[model_class][:1 if profile == "workload-smoke" else 2]) + ), "cadences": cadences, "artifact": artifact, "draft_artifact": draft, @@ -458,6 +524,7 @@ def _normalize_models(value: object, policy: dict[str, Any]) -> list[dict[str, A "startup_timeout_secs": startup_timeout_secs, }, "notes": notes, + **({"evidence": evidence} if evidence is not None else {}), "manifest_index": index, } ) @@ -631,6 +698,10 @@ def build_plan( "manifest": manifest_source, "manifest_sha256": manifest_sha256, "required_certification_lanes": list(CORE_LANES), + "model_class_lanes": { + model_class: list(lanes) for model_class, lanes in MODEL_CLASS_LANES.items() + }, + "requested_families": families or None, "selected_cadence": cadence or None, "selected_family_count": len(models), "selected_models": models, @@ -657,11 +728,55 @@ def _parse_args(argv: list[str]) -> argparse.Namespace: parser.add_argument("--check-cache", action="store_true") parser.add_argument("--output", type=Path) parser.add_argument("--github-output", type=Path) + parser.add_argument("--verify-plan", type=Path, help="recompute and verify an existing policy plan") + parser.add_argument( + "--inspect-gguf", + type=Path, + help="print the canonical layer count and activation width for one GGUF", + ) return parser.parse_args(argv) def main(argv: list[str] | None = None) -> int: args = _parse_args(sys.argv[1:] if argv is None else argv) + if args.inspect_gguf is not None: + dimensions = _gguf_dimensions(args.inspect_gguf) + if dimensions is None: + raise PlanError( + f"GGUF has no positive *.block_count and *.embedding_length metadata: {args.inspect_gguf}" + ) + layer_count, activation_width = dimensions + sys.stdout.write( + json.dumps( + { + "layer_count": layer_count, + "activation_width": activation_width, + }, + sort_keys=True, + ) + + "\n" + ) + return 0 + if args.verify_plan is not None: + supplied = _object(json.loads(args.verify_plan.read_text(encoding="utf-8")), "plan") + requested_families = supplied.get("requested_families") + if requested_families is not None and not isinstance(requested_families, str): + raise PlanError("plan.requested_families must be a string or null") + selected_cadence = supplied.get("selected_cadence") + if selected_cadence is not None and selected_cadence not in CADENCES: + raise PlanError("plan.selected_cadence is unsupported") + shards = supplied.get("shards") + if not isinstance(shards, list) or not shards: + raise PlanError("plan.shards must be a nonempty list") + expected = build_plan( + args.manifest, + families=requested_families or "", + cadence=selected_cadence or "", + shard_count=len(shards), + ) + if supplied != expected: + raise PlanError("policy plan differs from the canonical manifest and selection") + return 0 cache_root = args.cache_root if args.check_cache and cache_root is None: env_cache = os.environ.get("HF_CACHE") or os.environ.get("HF_HOME") diff --git a/scripts/skippy-family-battery.sh b/scripts/skippy-family-battery.sh index 5292512c6b..cf0aa191ca 100755 --- a/scripts/skippy-family-battery.sh +++ b/scripts/skippy-family-battery.sh @@ -3,13 +3,15 @@ set -euo pipefail # Supported-families certification battery (issue #1434; tiers dropped 2026-08-25). # -# Every row of the single manifest gets core certification: single-step, -# chain, and state-handoff lanes. Models with MTP/NextN tensors require the -# native draft sideband and verify it against the target in the correctness -# lanes. Dense rows run them at the first, midpoint, and last interior cuts. -# Hybrid/recurrent rows (sweep_period > 0) run a boundary sweep — one -# representative split layer for every cut offset modulo the family's -# interleaving period. +# Causal-generation rows get core split certification: single-step, chain, +# and state-handoff lanes. Non-chat rows get one provisional class-specific +# smoke lane through the Skippy runtime and OpenAI-compatible frontend; +# these classes deliberately fail closed if asked to stage. Models with +# MTP/NextN tensors require the native draft sideband and verify it against the +# target in the correctness lanes. Dense causal rows run them at the first, +# midpoint, and last interior cuts. Hybrid/recurrent rows (sweep_period > 0) +# run a boundary sweep — one representative split layer for every cut offset +# modulo the family's interleaving period. # # Models are NEVER cached through GitHub Actions cache. The family-certify # runner ships a large pre-warmed, read-only HF cache. When HF_CACHE is set, @@ -121,9 +123,11 @@ require_cmd() { } } -require_cmd hf require_cmd jq require_cmd python3 +if (( DRY_RUN == 0 )); then + require_cmd hf +fi if [[ ! -x "$PLANNER" ]]; then echo "family battery planner is not executable: $PLANNER" >&2 exit 1 @@ -159,7 +163,7 @@ fi mkdir -p "$MODEL_SCAN_DIR" "$PREFLIGHT_DIR" "$CERT_DIR" : > "$RESULTS_JSONL" printf 'family\tmodel_id\tsource_revision\tmodel_path\tmtp_layers\n' > "$NATIVE_MTP_MODELS_TSV" -printf 'family|repo|source_revision|file|selector|sweep_period|layer_end|notes|target_path|draft_repo|draft_revision|draft_file|draft_path|native_mtp|model_size_bytes|mtp_layers|activation_width|startup_timeout_secs|mmproj_repo|mmproj_revision|mmproj_file|mmproj_path\n' > "$RESOLVED_MANIFEST" +printf 'family|class|repo|source_revision|file|selector|sweep_period|layer_end|notes|target_path|draft_repo|draft_revision|draft_file|draft_path|native_mtp|model_size_bytes|mtp_layers|activation_width|startup_timeout_secs|lane_csv|mmproj_repo|mmproj_revision|mmproj_file|mmproj_path\n' > "$RESOLVED_MANIFEST" prepare_policy_plan() { local plan_args=( @@ -188,6 +192,7 @@ prepare_policy_plan() { "${plan_args[@]}" fi + "$PLANNER" --manifest "$MANIFEST" --verify-plan "$POLICY_PLAN_COPY" python3 - "$MANIFEST" "$POLICY_PLAN_COPY" "$SHARD_INDEX" <<'PY' import hashlib import json @@ -198,12 +203,23 @@ manifest_path, plan_path, shard_index = sys.argv[1:] manifest_sha = hashlib.sha256(Path(manifest_path).read_bytes()).hexdigest() plan = json.loads(Path(plan_path).read_text(encoding="utf-8")) core = ["single-step", "chain", "state-handoff"] +class_lanes = { + "causal_generation": core, + "embedding": ["embedding-smoke", "embedding-oracle"], + "rerank": ["rerank-smoke", "rerank-oracle"], + "encoder_decoder": ["encoder-decoder-smoke", "encoder-decoder-oracle"], + "ocr": ["ocr-smoke", "ocr-oracle"], + "speech_synthesis": ["speech-synthesis-smoke", "speech-synthesis-oracle"], + "speech_recognition": ["speech-recognition-smoke", "speech-recognition-oracle"], +} if plan.get("schema_version") != 1: raise SystemExit("policy plan has an unsupported schema_version") if plan.get("manifest_sha256") != manifest_sha: raise SystemExit("policy plan does not match the checked-in manifest bytes") if plan.get("required_certification_lanes") != core: raise SystemExit("policy plan does not preserve the three-lane certification contract") +if plan.get("model_class_lanes") != class_lanes: + raise SystemExit("policy plan does not preserve the model-class lane contract") if not plan.get("selected_models"): raise SystemExit("policy plan selected no models") if shard_index: @@ -358,6 +374,7 @@ scan_model() { MODEL_SIZE_BYTES=0 MODEL_MTP_LAYERS="" MODEL_LAYER_COUNT=0 + MODEL_ACTIVATION_WIDTH=0 if (( DRY_RUN == 1 )); then echo "$BIN_DIR/skippy-model-package inspect '$target' > '$scan_json'" @@ -376,7 +393,19 @@ scan_model() { fi MODEL_SIZE_BYTES="$(jq '[.tensors[].byte_size] | add // 0' "$scan_json")" - MODEL_LAYER_COUNT="$(jq '[.tensors[] | select(.layer_index != null) | .layer_index] | unique | length' "$scan_json")" + local dimensions + if ! dimensions="$("$PLANNER" --inspect-gguf "$target")"; then + jq -n \ + --arg family "$family" \ + --arg model_id "$model_id" \ + --arg target "$target" \ + '{family:$family,model_id:$model_id,target_model:$target,exit_code:1,outcomes:[{name:"model-metadata",status:"fail",outcome:"model-invalid",note:"could not read canonical GGUF dimensions"}]}' \ + >> "$RESULTS_JSONL" + FAILURES+=("$family(metadata)") + return 1 + fi + MODEL_LAYER_COUNT="$(jq -r '.layer_count' <<<"$dimensions")" + MODEL_ACTIVATION_WIDTH="$(jq -r '.activation_width' <<<"$dimensions")" MODEL_MTP_LAYERS="$(jq -r ' [.tensors[] | select(.layer_index != null) @@ -400,14 +429,22 @@ scan_model() { preflight_environment() { local model_root="${HF_HOME:-$(dirname "$PREFLIGHT_FIRST_TARGET")}" - python3 - "$ARTIFACT_DIR" "$model_root" "$MIN_FREE_GIB" "$PREFLIGHT_DIR/environment.json" <<'PY' + local port_mode="full" needs_oracle_server=0 + if ! jq -e '[.selected_models[].class] | any(. == "causal_generation")' "$POLICY_PLAN_COPY" >/dev/null; then + port_mode="workload" + if jq -e '[.selected_models[].class] | any(. == "embedding" or . == "rerank" or . == "ocr" or . == "speech_recognition")' "$POLICY_PLAN_COPY" >/dev/null; then + needs_oracle_server=1 + fi + fi + python3 - "$ARTIFACT_DIR" "$model_root" "$MIN_FREE_GIB" "$PREFLIGHT_ONLY" "$PREFLIGHT_DIR/environment.json" \ + "$port_mode" "${SKIPPY_WORKLOAD_OPENAI_PORT:-19337}" "${SKIPPY_WORKLOAD_ORACLE_PORT:-19338}" "$needs_oracle_server" <<'PY' import json import shutil import socket import sys from pathlib import Path -artifact_root, model_root, minimum_gib, output = sys.argv[1:] +artifact_root, model_root, minimum_gib, preflight_only, output, port_mode, candidate_port, oracle_port, needs_oracle_server = sys.argv[1:] minimum_bytes = int(minimum_gib) * 1024**3 filesystems = [] for label, path_text in (("artifacts", artifact_root), ("models", model_root)): @@ -426,20 +463,32 @@ for label, path_text in (("artifacts", artifact_root), ("models", model_root)): ) busy_ports = [] -for port in range(19000, 20032): - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - try: - sock.settimeout(0.01) - if sock.connect_ex(("127.0.0.1", port)) == 0: +ports = list(range(19000, 20032)) if port_mode == "full" else [int(candidate_port)] +if port_mode == "workload" and needs_oracle_server == "1": + ports.append(int(oracle_port)) +if any(port < 1 or port > 65535 for port in ports) or len(ports) != len(set(ports)): + raise SystemExit("invalid or conflicting workload certification ports") +if preflight_only == "0": + for port in ports: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + sock.settimeout(0.01) + if sock.connect_ex(("127.0.0.1", port)) == 0: + busy_ports.append(port) + except OSError: busy_ports.append(port) - except OSError: - busy_ports.append(port) - finally: - sock.close() + finally: + sock.close() report = { "filesystems": filesystems, - "port_range": {"start": 19000, "end": 20031, "busy": busy_ports}, + "port_range": { + "start": min(ports), + "end": max(ports), + "ports_checked": ports if port_mode == "workload" else None, + "checked": preflight_only == "0", + "busy": busy_ports, + }, } Path(output).write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") if any(not item["sufficient"] for item in filesystems) or busy_ports: @@ -549,13 +598,17 @@ preflight_manifest() { return 1 fi - while IFS='|' read -r family profile repo source_revision file selector sweep_period layer_end activation_width notes draft_repo draft_revision draft_file expected_model_bytes startup_timeout_override expected_mtp_layers lane_csv _speculative_policy mmproj_repo mmproj_revision mmproj_file; do - if [[ "$profile" != "full" ]]; then - echo "the local monolithic battery cannot execute profile $profile for $family" >&2 + while IFS='|' read -r family model_class profile repo source_revision file selector sweep_period layer_end activation_width notes draft_repo draft_revision draft_file expected_model_bytes startup_timeout_override expected_mtp_layers lane_csv _speculative_policy mmproj_repo mmproj_revision mmproj_file; do + if [[ "$model_class" == "causal_generation" && "$profile" != "full" ]] || + [[ "$model_class" != "causal_generation" && "$profile" != "workload-smoke" && "$profile" != "workload-oracle" ]]; then + echo "the family battery cannot execute profile $profile for $model_class family $family" >&2 exit 1 fi - if [[ "$lane_csv" != "single-step,chain,state-handoff" ]]; then - echo "certified family $family does not preserve the three-lane contract" >&2 + local expected_lane_csv + expected_lane_csv="$(jq -r --arg model_class "$model_class" --arg profile "$profile" \ + '.model_class_lanes[$model_class] | if $profile == "workload-smoke" then .[:1] else . end | join(",")' "$plan")" + if [[ -z "$expected_lane_csv" || "$lane_csv" != "$expected_lane_csv" ]]; then + echo "family $family does not preserve the $model_class lane contract" >&2 exit 1 fi @@ -594,6 +647,13 @@ preflight_manifest() { record_preflight_outcome "model-preflight" "$family" "$model_id" "fail" "model-invalid" "planned runtime range $layer_end does not match scanned layer count $MODEL_LAYER_COUNT" continue fi + if (( MODEL_ACTIVATION_WIDTH != activation_width )); then + echo "policy/runtime activation-width mismatch for $family: planned $activation_width, scanned $MODEL_ACTIVATION_WIDTH" >&2 + FAILURES+=("$family(activation-width)") + PREFLIGHT_FAILURE_COUNT=$((PREFLIGHT_FAILURE_COUNT + 1)) + record_preflight_outcome "model-preflight" "$family" "$model_id" "fail" "model-invalid" "planned activation width $activation_width does not match GGUF metadata $MODEL_ACTIVATION_WIDTH" + continue + fi if (( actual_mtp_layers != expected_mtp_layers )); then echo "policy/MTP mismatch for $family: planned $expected_mtp_layers, scanned $actual_mtp_layers" >&2 FAILURES+=("$family(mtp-policy)") @@ -635,9 +695,9 @@ preflight_manifest() { mmproj_path="/$mmproj_repo/$mmproj_file" fi fi - printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \ - "$family" "$repo" "$source_revision" "$file" "$selector" "$sweep_period" "$layer_end" "$notes" "$target" \ - "$draft_repo" "$draft_revision" "$draft_file" "$draft" "$MODEL_HAS_MTP" "$MODEL_SIZE_BYTES" "$MODEL_MTP_LAYERS" "$activation_width" "$startup_timeout" \ + printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \ + "$family" "$model_class" "$repo" "$source_revision" "$file" "$selector" "$sweep_period" "$layer_end" "$notes" "$target" \ + "$draft_repo" "$draft_revision" "$draft_file" "$draft" "$MODEL_HAS_MTP" "$MODEL_SIZE_BYTES" "$MODEL_MTP_LAYERS" "$activation_width" "$startup_timeout" "$lane_csv" \ "$mmproj_repo" "$mmproj_revision" "$mmproj_file" "$mmproj_path" \ >> "$RESOLVED_MANIFEST" if (( DRY_RUN == 0 )); then @@ -656,6 +716,7 @@ preflight_manifest() { | select(.family as $family | $selected | index($family)) | [ .family, + .class, .profile, .artifact.repo, .artifact.revision, @@ -709,19 +770,31 @@ preflight_manifest() { return 1 fi if ! preflight_environment; then - record_preflight_outcome "environment-preflight" "battery" "environment" "fail" "harness" "insufficient disk headroom or occupied certification ports; see preflight/environment.json" + local environment_failure_note="insufficient disk headroom; see preflight/environment.json" + if (( PREFLIGHT_ONLY == 0 )); then + environment_failure_note="insufficient disk headroom or occupied certification ports; see preflight/environment.json" + fi + record_preflight_outcome "environment-preflight" "battery" "environment" "fail" "harness" "$environment_failure_note" PREFLIGHT_FAILURE_COUNT=$((PREFLIGHT_FAILURE_COUNT + 1)) return 1 fi - record_preflight_outcome "environment-preflight" "battery" "environment" "pass" "pass" "disk headroom and certification port range validated" + local environment_note="disk headroom validated; certification ports not checked in preflight-only mode" + if (( PREFLIGHT_ONLY == 0 )); then + environment_note="disk headroom and certification port range validated" + fi + record_preflight_outcome "environment-preflight" "battery" "environment" "pass" "pass" "$environment_note" } planned_certification_count() { local resolved_manifest="$1" local planned=0 - while IFS='|' read -r family _repo _source_revision _file _selector sweep_period layer_end _rest; do + while IFS='|' read -r family model_class _repo _source_revision _file _selector sweep_period layer_end _rest; do [[ "$family" == "family" ]] && continue + if [[ "$model_class" != "causal_generation" ]]; then + planned=$((planned + 1)) + continue + fi local base_split=$(( layer_end / 2 )) local first_split=1 local last_split=$(( layer_end - 1 )) @@ -796,12 +869,119 @@ run_mmproj_smoke() { >> "$RESULTS_JSONL" } +run_workload_certify() { + local family="$1" model_class="$2" target="$3" model_id="$4" source_revision="$5" + local startup_timeout="$6" model_size_bytes="$7" lane_csv="$8" mmproj="$9" + local cert_run_id cert_run_dir exit_code log_path cert_timeout oracle_requested + local smoke_lane oracle_lane certified oracle_executable + TOTAL=$((TOTAL + 1)) + cert_timeout="$(cert_timeout_for_startup "$startup_timeout")" + cert_run_id="$(printf '%03d-%s-%s' "$TOTAL" "$(slugify "$family")" "$(slugify "$model_class")")" + cert_run_dir="$CERT_DIR/$cert_run_id" + mkdir -p "$cert_run_dir" + log_path="$cert_run_dir/workload-certification.log" + smoke_lane="${lane_csv%%,*}" + oracle_lane="" + certified=0 + if [[ "$lane_csv" == *,* ]]; then + oracle_lane="${lane_csv#*,}" + certified=1 + fi + local command=( + "$ROOT/scripts/skippy-workload-certify.sh" + --class "$model_class" + --lane "$smoke_lane" + --model-path "$target" + --model-id "$model_id" + --work-dir "$cert_run_dir" + --skip-build + ) + if [[ -n "$mmproj" ]]; then + command+=(--projector-path "$mmproj") + fi + oracle_requested=0 + oracle_executable="" + if [[ -n "${SKIPPY_WORKLOAD_ORACLE_SERVER:-}" ]] && + [[ "$model_class" =~ ^(embedding|rerank|ocr|speech_recognition)$ ]]; then + command+=(--oracle-server "$SKIPPY_WORKLOAD_ORACLE_SERVER") + oracle_requested=1 + oracle_executable="$SKIPPY_WORKLOAD_ORACLE_SERVER" + fi + if [[ -n "${SKIPPY_WORKLOAD_ORACLE_COMPLETION:-}" ]] && + [[ "$model_class" == "encoder_decoder" ]]; then + command+=(--oracle-completion "$SKIPPY_WORKLOAD_ORACLE_COMPLETION") + oracle_requested=1 + oracle_executable="$SKIPPY_WORKLOAD_ORACLE_COMPLETION" + fi + if [[ -n "${SKIPPY_WORKLOAD_ORACLE_TTS:-}" ]] && + [[ "$model_class" == "speech_synthesis" ]]; then + command+=(--oracle-tts "$SKIPPY_WORKLOAD_ORACLE_TTS") + oracle_requested=1 + oracle_executable="$SKIPPY_WORKLOAD_ORACLE_TTS" + fi + if (( certified == 1 )); then + if (( oracle_requested != 1 )); then + echo "certified workload $family ($model_class) requires a class-appropriate local-monolithic oracle executable" >&2 + exit 1 + fi + command+=(--require-oracle) + fi + echo "==> workload certification: family=$family class=$model_class lanes=$lane_csv model=$(basename "$target")" + if (( DRY_RUN == 1 )); then + printf '%q ' "${command[@]}" + printf '\n' + return 0 + fi + exit_code=0 + "$ROOT/scripts/run-command-with-timeout.py" \ + --seconds "$cert_timeout" \ + --label "workload certification $family ($model_class)" \ + -- "${command[@]}" >"$log_path" 2>&1 || exit_code=$? + if (( certified == 1 && exit_code == 0 )); then + local verify_command=(python3 "$ROOT/scripts/verify-workload-oracle-evidence.py" \ + --evidence "$cert_run_dir/workload-oracle-evidence.json" \ + --class "$model_class" --smoke-lane "$smoke_lane" --oracle-lane "$oracle_lane" \ + --model-id "$model_id" --model-path "$target" \ + --candidate-executable "$ROOT/target/debug/skippy-server" \ + --oracle-executable "$oracle_executable" \ + --pinned-patch-sha "$(python3 "$ROOT/scripts/llama-oracle-source.py")") + if [[ -n "$mmproj" ]]; then + verify_command+=(--projector-path "$mmproj") + fi + "${verify_command[@]}" >>"$log_path" 2>&1 || exit_code=$? + fi + jq -n \ + --arg family "$family" \ + --arg model_id "$model_id" \ + --arg source_revision "$source_revision" \ + --arg workload_class "$model_class" \ + --arg smoke_lane "$smoke_lane" \ + --arg oracle_lane "$oracle_lane" \ + --arg log "$log_path" \ + --argjson model_size_bytes "$model_size_bytes" \ + --argjson startup_timeout_secs "$startup_timeout" \ + --argjson certification_timeout_secs "$cert_timeout" \ + --argjson exit_code "$exit_code" \ + --argjson oracle_requested "$oracle_requested" \ + '{family:$family,model_id:$model_id,source_revision:$source_revision,workload_class:$workload_class,certification_status:(if $oracle_lane != "" then "certified" else "provisional" end),oracle:(if $oracle_lane != "" and $exit_code == 0 then "local-monolithic" else "none" end),model_size_bytes:$model_size_bytes,startup_timeout_secs:$startup_timeout_secs,certification_timeout_secs:$certification_timeout_secs,exit_code:$exit_code,outcomes:([{name:$smoke_lane,status:(if $exit_code == 0 then "pass" else "fail" end),outcome:(if $exit_code == 0 then "smoke-pass" elif $exit_code == 124 then "timeout" else "harness" end),exit_code:$exit_code,log:$log}] + (if $oracle_lane == "" then [] else [{name:$oracle_lane,status:(if $exit_code == 0 then "pass" else "fail" end),outcome:(if $exit_code == 0 then "oracle-pass" elif $exit_code == 124 then "timeout" else "harness" end),exit_code:$exit_code,log:$log}] end))}' \ + >> "$RESULTS_JSONL" + if (( exit_code != 0 )); then + FAILURES+=("$family@$model_class") + CERT_FAILURE_COUNT=$((CERT_FAILURE_COUNT + 1)) + fi +} + run_resolved_manifest() { local resolved_manifest="$1" - while IFS='|' read -r family repo source_revision file selector sweep_period layer_end _notes target _draft_repo _draft_revision _draft_file _draft native_mtp model_size_bytes _mtp_layers activation_width startup_timeout mmproj_repo mmproj_revision mmproj_file mmproj_path; do + while IFS='|' read -r family model_class repo source_revision file selector sweep_period layer_end _notes target _draft_repo _draft_revision _draft_file _draft native_mtp model_size_bytes _mtp_layers activation_width startup_timeout lane_csv mmproj_repo mmproj_revision mmproj_file mmproj_path; do [[ "$family" == "family" ]] && continue local model_id="$repo:$selector" + if [[ "$model_class" != "causal_generation" ]]; then + run_workload_certify "$family" "$model_class" "$target" "$model_id" "$source_revision" "$startup_timeout" "$model_size_bytes" "$lane_csv" "$mmproj_path" + continue + fi + # Dense families exercise both endpoint ownership cases plus an ordinary # interior handoff. Collapse duplicates for tiny models. local base_split=$(( layer_end / 2 )) @@ -841,7 +1021,7 @@ if ! preflight_manifest "$POLICY_PLAN_COPY"; then echo "family battery preflight failed; no certification lane was started" >&2 elif (( PREFLIGHT_ONLY == 0 )); then EXPECTED_TOTAL="$(planned_certification_count "$RESOLVED_MANIFEST")" - EXPECTED_MM_SMOKE_TOTAL="$(tail -n +2 "$RESOLVED_MANIFEST" | awk -F'|' '$19 != "" { count += 1 } END { print count + 0 }')" + EXPECTED_MM_SMOKE_TOTAL="$(tail -n +2 "$RESOLVED_MANIFEST" | awk -F'|' '$2 == "causal_generation" && $21 != "" { count += 1 } END { print count + 0 }')" run_resolved_manifest "$RESOLVED_MANIFEST" if (( TOTAL != EXPECTED_TOTAL )); then echo "executed $TOTAL certifications but validated plan requires $EXPECTED_TOTAL" >&2 @@ -854,7 +1034,7 @@ elif (( PREFLIGHT_ONLY == 0 )); then CERT_FAILURE_COUNT=$((CERT_FAILURE_COUNT + 1)) fi if (( DRY_RUN == 0 )); then - actual_result_count="$(jq -s '[.[] | select(.split_layer != null)] | length' "$RESULTS_JSONL")" + actual_result_count="$(jq -s '[.[] | select(.split_layer != null or .workload_class != null)] | length' "$RESULTS_JSONL")" if (( actual_result_count != EXPECTED_TOTAL )); then echo "recorded $actual_result_count certification results but validated plan requires $EXPECTED_TOTAL" >&2 FAILURES+=("battery(result-reconciliation)") @@ -866,8 +1046,8 @@ fi echo if (( DRY_RUN == 0 )); then jq -sr ' - ["family","split_layer","lane","status","outcome","exit_code"], - (.[] as $row | $row.outcomes[] | [$row.family,($row.split_layer // ""),.name,.status,.outcome,.exit_code]) + ["family","class","split_layer","lane","status","outcome","exit_code"], + (.[] as $row | $row.outcomes[] | [$row.family,($row.workload_class // "causal_generation"),($row.split_layer // ""),.name,.status,.outcome,.exit_code]) | @tsv ' "$RESULTS_JSONL" > "$SUMMARY_TSV" { @@ -880,6 +1060,9 @@ if (( DRY_RUN == 0 )); then fi echo "- Certifications: $TOTAL" echo "- Planned certifications: $EXPECTED_TOTAL" + echo "- Certified-profile families: $(jq '[.selected_models[] | select(.certification_status == "certified")] | length' "$POLICY_PLAN_COPY")" + echo "- Provisional workload-smoke families: $(jq '[.selected_models[] | select(.profile == "workload-smoke")] | length' "$POLICY_PLAN_COPY")" + echo "- Certified non-chat workloads require separate smoke and explicit local-monolithic oracle evidence." echo "- Multimodal smokes: $MM_SMOKE_TOTAL (planned: $EXPECTED_MM_SMOKE_TOTAL; failures: $MM_SMOKE_FAILURE_COUNT)" echo "- Native MTP models: $(( $(wc -l < "$NATIVE_MTP_MODELS_TSV") - 1 ))" echo "- Preflight failures: $PREFLIGHT_FAILURE_COUNT" @@ -905,6 +1088,8 @@ fi if (( PREFLIGHT_ONLY == 1 )); then echo "family battery preflight complete: $PREFLIGHT_FAILURE_COUNT failures" +elif (( DRY_RUN == 1 )); then + echo "family battery dry run complete: $TOTAL certifications planned; no lanes executed" else echo "family battery complete: $((TOTAL - CERT_FAILURE_COUNT))/$TOTAL certifications passed" fi diff --git a/scripts/skippy-ocr-asr-oracle.py b/scripts/skippy-ocr-asr-oracle.py new file mode 100644 index 0000000000..aefbb859a3 --- /dev/null +++ b/scripts/skippy-ocr-asr-oracle.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +"""Compare OCR or ASR output with pinned llama.cpp monolithic serving. + +Both HTTP servers must load the same GGUF and projector on CPU. Matching text +is a local execution-parity check, not general model-quality certification. +OCR additionally checks the known text in the generated fixture. ASR requires +an independently labeled audio fixture before semantic accuracy can be claimed. +""" + +from __future__ import annotations + +import argparse +import base64 +import json +from pathlib import Path +import re +import unicodedata +import urllib.request + + +OCR_PROMPT = "Read all visible text. Return only the transcription." +OCR_FIXTURE_TEXT = "MESH 42" +ASR_PROMPT = "Transcribe audio to text" +BOUNDARY = "mesh-llm-ocr-asr-oracle" +NON_TRANSCRIPT_PREFIXES = ( + "i can t fulfill", + "i cannot fulfill", + "i can help you with transcribing", + "you can use various tools to transcribe", +) + + +def normalized_text(value: object, source: str) -> str: + if not isinstance(value, str): + raise RuntimeError(f"{source} returned no text") + normalized = unicodedata.normalize("NFKC", value).casefold() + normalized = re.sub(r"[^\w]+", " ", normalized, flags=re.UNICODE).strip() + if not normalized: + raise RuntimeError(f"{source} returned empty text") + return normalized + + +def transcription_text(value: object, source: str) -> str: + text = normalized_text(value, source) + # The two frontends add different presentational labels around the same + # transcript. Strip only these exact known prefixes, never content words. + for prefix in ("the text is ", "the audio is "): + if text.startswith(prefix): + text = text[len(prefix):] + break + if text.startswith(NON_TRANSCRIPT_PREFIXES): + raise RuntimeError(f"{source} returned a refusal or generic ASR advice, not a transcript") + return text + + +def compare_text(candidate: object, reference: object, expected: str | None, + *, transcript: bool = False) -> str: + normalizer = transcription_text if transcript else normalized_text + candidate_text = normalizer(candidate, "candidate") + reference_text = normalizer(reference, "monolithic reference") + if candidate_text != reference_text: + raise RuntimeError( + "text differs from monolithic reference: " + f"candidate={candidate_text!r}, reference={reference_text!r}" + ) + if expected is not None: + expected_text = normalized_text(expected, "fixture label") + if expected_text not in candidate_text: + raise RuntimeError( + "output misses independently known fixture text: " + f"expected={expected_text!r}, actual={candidate_text!r}" + ) + return f"identical normalized text containing {expected_text!r}" + return "identical normalized text; unlabeled fixture, no accuracy claim" + + +def request_json(base_url: str, path: str, payload: dict[str, object]) -> dict: + request = urllib.request.Request( + f"{base_url.rstrip('/')}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + return response_json(request) + + +def request_multipart(base_url: str, path: str, model: str, media: bytes) -> dict: + if BOUNDARY.encode("ascii") in media: + raise RuntimeError("audio fixture collides with multipart boundary") + body = ( + f"--{BOUNDARY}\r\n" + 'Content-Disposition: form-data; name="model"\r\n\r\n' + f"{model}\r\n" + f"--{BOUNDARY}\r\n" + 'Content-Disposition: form-data; name="response_format"\r\n\r\n' + "json\r\n" + f"--{BOUNDARY}\r\n" + 'Content-Disposition: form-data; name="temperature"\r\n\r\n' + "0\r\n" + f"--{BOUNDARY}\r\n" + 'Content-Disposition: form-data; name="file"; filename="oracle.wav"\r\n' + "Content-Type: audio/wav\r\n\r\n" + ).encode("utf-8") + media + f"\r\n--{BOUNDARY}--\r\n".encode("ascii") + request = urllib.request.Request( + f"{base_url.rstrip('/')}{path}", + data=body, + headers={"Content-Type": f"multipart/form-data; boundary={BOUNDARY}"}, + method="POST", + ) + return response_json(request) + + +def response_json(request: urllib.request.Request) -> dict: + with urllib.request.urlopen(request, timeout=240) as response: + if response.headers.get_content_type() != "application/json": + raise RuntimeError(f"{request.full_url} returned non-JSON content") + result = json.load(response) + if not isinstance(result, dict): + raise RuntimeError(f"{request.full_url} returned a non-object response") + return result + + +def chat_text(response: dict, source: str) -> object: + choices = response.get("choices") + if not isinstance(choices, list) or len(choices) != 1: + raise RuntimeError(f"{source} returned invalid OCR choices") + choice = choices[0] + if not isinstance(choice, dict): + raise RuntimeError(f"{source} returned an invalid OCR choice") + message = choice.get("message") + if not isinstance(message, dict): + raise RuntimeError(f"{source} returned no OCR message") + return message.get("content") + + +def compare_ocr(candidate_url: str, oracle_url: str, model: str, image: bytes, + expected: str) -> str: + payload = { + "model": model, + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": OCR_PROMPT}, + {"type": "image_url", "image_url": { + "url": "data:image/png;base64," + base64.b64encode(image).decode("ascii") + }}, + ], + }], + "max_tokens": 64, + "temperature": 0.0, + "seed": 1, + } + candidate = request_json(candidate_url, "/chat/completions", payload) + reference = request_json(oracle_url, "/chat/completions", payload) + return compare_text( + chat_text(candidate, "candidate"), + chat_text(reference, "monolithic reference"), + expected, + ) + + +def compare_asr(candidate_url: str, oracle_url: str, model: str, audio: bytes, + expected: str | None) -> str: + # llama-server's /audio/transcriptions substitutes its own default user + # instruction. Compare the actual Skippy audio route with monolithic chat + # using the exact instruction and media ordering that Skippy constructs. + candidate = request_multipart(candidate_url, "/audio/transcriptions", model, audio) + reference = request_json(oracle_url, "/chat/completions", { + "model": model, + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": ASR_PROMPT}, + {"type": "input_audio", "input_audio": { + "data": base64.b64encode(audio).decode("ascii"), "format": "wav" + }}, + ], + }], + "temperature": 0.0, + "max_tokens": 128, + }) + return compare_text( + candidate.get("text"), + chat_text(reference, "monolithic reference"), + expected, + transcript=True, + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidate-url", required=True) + parser.add_argument("--oracle-url", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--class", dest="model_class", required=True, + choices=("ocr", "speech_recognition")) + parser.add_argument("--media-path", required=True, type=Path) + parser.add_argument("--expected-text", help="independent label for the media fixture") + args = parser.parse_args() + + if not args.media_path.is_file(): + parser.error("--media-path must point to a readable fixture") + if args.model_class == "ocr" and args.expected_text is None: + expected = OCR_FIXTURE_TEXT + else: + expected = args.expected_text + media = args.media_path.read_bytes() + if args.model_class == "ocr": + detail = compare_ocr(args.candidate_url, args.oracle_url, args.model, media, expected) + else: + detail = compare_asr(args.candidate_url, args.oracle_url, args.model, media, expected) + print(f"{args.model_class} local-monolithic oracle passed: {detail}") + + +if __name__ == "__main__": + main() diff --git a/scripts/skippy-tts-oracle.py b/scripts/skippy-tts-oracle.py new file mode 100644 index 0000000000..8fa61efbec --- /dev/null +++ b/scripts/skippy-tts-oracle.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 +"""Compare deterministic Skippy TTS PCM with pinned llama-tts full-model PCM. + +The public speech endpoint uses a random seed, so this numerical oracle runs a +test-only in-process candidate with shared fixed sampling and the same GGUF, +projector, and prompt as llama-tts. The regular HTTP smoke remains a separate +API contract check; waveform parity does not certify intelligibility. +""" + +from __future__ import annotations + +import argparse +from array import array +import hashlib +import json +import math +import os +from pathlib import Path +import subprocess +import sys +import wave + + +ROOT = Path(__file__).resolve().parents[1] +PROMPT = "The mesh is ready." +SEED = 7 +TOP_K = 20 +TOP_P = 0.8 +MAX_FRAMES = 32 +CONTEXT_SIZE = 2048 +MAX_RELATIVE_RMS_ERROR = 0.02 +MIN_WAVEFORM_COSINE = 0.9995 +TEST_NAME = "frontend::tests::tts_oracle::deterministic_tts_candidate_when_fixture_is_set" + + +def require_pinned_cpu_oracle(oracle_cli: Path) -> str: + if ( + oracle_cli.name != "llama-tts" + or not oracle_cli.is_file() + or not os.access(oracle_cli, os.X_OK) + ): + raise RuntimeError("oracle CLI must be an executable llama-tts binary") + stamp_path = oracle_cli.parent.parent / ".mesh-llm-build-stamp" + patched_sha_path = ROOT / ".deps/llama.cpp/.mesh-llm-patched-sha" + if not stamp_path.is_file() or not patched_sha_path.is_file(): + raise RuntimeError("llama-tts lacks the current pinned build stamp") + stamp = set(stamp_path.read_text(encoding="utf-8").splitlines()) + patched_sha = patched_sha_path.read_text(encoding="utf-8").strip() + required = { + f"patched-sha={patched_sha}", + "backend=cpu", + "link-mode=static", + "ggml-native=OFF", + "cmake-arg=-DGGML_NATIVE=OFF", + "cmake-arg=-DGGML_METAL=OFF", + "cmake-arg=-DLLAMA_BUILD_TOOLS=ON", + } + if not patched_sha or not required.issubset(stamp): + raise RuntimeError("llama-tts lacks the current pinned CPU build stamp") + return patched_sha + + +def require_candidate_cpu_static_build(build_dir: Path, patched_sha: str) -> None: + stamp_path = build_dir / ".mesh-llm-build-stamp" + if not stamp_path.is_file(): + raise RuntimeError("TTS candidate lacks a pinned static CPU build stamp") + stamp = set(stamp_path.read_text(encoding="utf-8").splitlines()) + required = { + f"patched-sha={patched_sha}", + "backend=cpu", + "link-mode=static", + "ggml-native=OFF", + "cmake-arg=-DGGML_NATIVE=OFF", + "cmake-arg=-DGGML_METAL=OFF", + } + if not required.issubset(stamp): + raise RuntimeError("TTS candidate static CPU build does not match the oracle patch SHA") + + +def run_logged(command: list[str], log_path: Path, *, env: dict[str, str] | None = None) -> None: + with log_path.open("w", encoding="utf-8") as log: + try: + result = subprocess.run( + command, + cwd=ROOT, + env=env, + stdout=log, + stderr=subprocess.STDOUT, + timeout=900, + check=False, + ) + except subprocess.TimeoutExpired as error: + raise RuntimeError(f"oracle command timed out; see {log_path}") from error + if result.returncode != 0: + tail = "\n".join(log_path.read_text(encoding="utf-8", errors="replace").splitlines()[-30:]) + raise RuntimeError(f"oracle command exited {result.returncode}; see {log_path}\n{tail}") + + +def read_pcm16_wav(path: Path) -> tuple[int, int, array]: + try: + with wave.open(str(path), "rb") as audio: + sample_rate = audio.getframerate() + channels = audio.getnchannels() + if audio.getcomptype() != "NONE" or audio.getsampwidth() != 2: + raise RuntimeError(f"{path.name} is not uncompressed PCM16 WAV") + raw = audio.readframes(audio.getnframes()) + except (OSError, EOFError, wave.Error) as error: + raise RuntimeError(f"cannot read {path.name} as WAV: {error}") from error + if sample_rate <= 0 or channels <= 0 or not raw or len(raw) % (2 * channels): + raise RuntimeError(f"{path.name} has invalid or empty PCM samples") + samples = array("h") + samples.frombytes(raw) + if sys.byteorder != "little": + samples.byteswap() + return sample_rate, channels, samples + + +def compare_wavs(candidate_path: Path, oracle_path: Path) -> dict[str, float | int]: + candidate_rate, candidate_channels, candidate = read_pcm16_wav(candidate_path) + oracle_rate, oracle_channels, oracle = read_pcm16_wav(oracle_path) + if candidate_rate != oracle_rate or candidate_channels != oracle_channels: + raise RuntimeError("TTS sample rate or channel count differs from monolithic oracle") + if len(candidate) != len(oracle): + raise RuntimeError( + "TTS sample count differs from monolithic oracle: " + f"candidate={len(candidate)}, reference={len(oracle)}" + ) + candidate_energy = math.fsum(float(value) ** 2 for value in candidate) + oracle_energy = math.fsum(float(value) ** 2 for value in oracle) + if candidate_energy <= len(candidate) or oracle_energy <= len(oracle): + raise RuntimeError("TTS candidate or monolithic oracle is silent") + delta_energy = math.fsum( + float(left - right) ** 2 for left, right in zip(candidate, oracle, strict=True) + ) + dot = math.fsum( + float(left) * right for left, right in zip(candidate, oracle, strict=True) + ) + relative_rms_error = math.sqrt(delta_energy / oracle_energy) + waveform_cosine = dot / math.sqrt(candidate_energy * oracle_energy) + metrics: dict[str, float | int] = { + "sample_rate_hz": candidate_rate, + "channels": candidate_channels, + "sample_count": len(candidate) // candidate_channels, + "relative_rms_error": relative_rms_error, + "waveform_cosine": waveform_cosine, + } + if relative_rms_error > MAX_RELATIVE_RMS_ERROR or waveform_cosine < MIN_WAVEFORM_COSINE: + raise RuntimeError( + "TTS PCM differs from monolithic oracle: " + f"relative_rms_error={relative_rms_error:.7g}, waveform_cosine={waveform_cosine:.8g}" + ) + return metrics + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def run_oracle(args: argparse.Namespace) -> dict[str, object]: + oracle_cli = Path(args.oracle_cli).resolve() + model_path = Path(args.model_path).resolve() + projector_path = Path(args.projector_path).resolve() + work_dir = Path(args.work_dir).resolve() + patched_sha = require_pinned_cpu_oracle(oracle_cli) + candidate_build_dir = os.environ.get("LLAMA_STAGE_BUILD_DIR") + if not candidate_build_dir: + raise RuntimeError("LLAMA_STAGE_BUILD_DIR is required for pinned TTS candidate execution") + require_candidate_cpu_static_build(Path(candidate_build_dir).resolve(), patched_sha) + for label, path in (("model", model_path), ("projector", projector_path)): + if not path.is_file(): + raise RuntimeError(f"{label} path is not a file: {path}") + if args.layer_end < 1 or not args.model: + raise RuntimeError("model alias and positive layer count are required") + work_dir.mkdir(parents=True, exist_ok=True) + candidate_wav = work_dir / "tts-candidate.wav" + oracle_wav = work_dir / "tts-monolithic-oracle.wav" + result_path = work_dir / "tts-oracle-result.json" + # A failed or filtered test must not inherit a WAV or PASS record from an + # earlier invocation of the same work directory. + for stale_output in (candidate_wav, oracle_wav, result_path): + stale_output.unlink(missing_ok=True) + candidate_env = os.environ.copy() + candidate_env.update({ + "LLAMA_STAGE_BACKEND": "cpu", + "SKIPPY_WORKLOAD_MODEL": str(model_path), + "SKIPPY_WORKLOAD_PROJECTOR": str(projector_path), + "SKIPPY_WORKLOAD_MODEL_ID": args.model, + "SKIPPY_WORKLOAD_LAYER_END": str(args.layer_end), + "SKIPPY_TTS_ORACLE_CANDIDATE_WAV": str(candidate_wav), + "SKIPPY_TTS_ORACLE_PROMPT": PROMPT, + "SKIPPY_TTS_ORACLE_SEED": str(SEED), + "SKIPPY_TTS_ORACLE_TOP_K": str(TOP_K), + "SKIPPY_TTS_ORACLE_TOP_P": str(TOP_P), + "SKIPPY_TTS_ORACLE_MAX_FRAMES": str(MAX_FRAMES), + }) + run_logged( + ["cargo", "test", "--manifest-path", str(ROOT / "Cargo.toml"), + "-p", "skippy-server", "--lib", TEST_NAME, + "--", "--exact", "--nocapture", "--test-threads=1"], + work_dir / "tts-candidate-test.log", + env=candidate_env, + ) + if not candidate_wav.is_file(): + raise RuntimeError("deterministic TTS candidate test did not write WAV output") + # StageConfig leaves weight repacking disabled. llama-tts enables it by + # default, which changes Q8 logits enough to alter stochastic audio tokens. + run_logged( + [str(oracle_cli), "-m", str(model_path), "-mm", str(projector_path), + "-p", PROMPT, "--output", str(oracle_wav), + "-n", str(MAX_FRAMES), "--seed", str(SEED), + "--top-k", str(TOP_K), "--top-p", str(TOP_P), + "--temp", "1", "--min-p", "0", "--repeat-penalty", "1", + "--no-repack", + "-c", str(CONTEXT_SIZE), "-b", str(CONTEXT_SIZE), + "-ub", str(CONTEXT_SIZE), "-ngl", "0"], + work_dir / "tts-monolithic-oracle.log", + ) + metrics = compare_wavs(candidate_wav, oracle_wav) + result: dict[str, object] = { + "status": "pass", + "class": "speech_synthesis", + "mode": "deterministic_local_monolithic_pcm_parity", + "prompt": PROMPT, + "seed": SEED, + "top_k": TOP_K, + "top_p": TOP_P, + "max_frames": MAX_FRAMES, + "pinned_patch_sha": patched_sha, + "thresholds": { + "max_relative_rms_error": MAX_RELATIVE_RMS_ERROR, + "min_waveform_cosine": MIN_WAVEFORM_COSINE, + }, + "metrics": metrics, + "candidate_wav_sha256": sha256(candidate_wav), + "oracle_wav_sha256": sha256(oracle_wav), + } + result_path.write_text( + json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return result + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--oracle-cli", required=True) + parser.add_argument("--model-path", required=True) + parser.add_argument("--projector-path", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--layer-end", type=int, required=True) + parser.add_argument("--work-dir", required=True) + result = run_oracle(parser.parse_args()) + print( + "speech_synthesis local-monolithic oracle passed: " + + json.dumps(result["metrics"], sort_keys=True) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/skippy-workload-certify.sh b/scripts/skippy-workload-certify.sh new file mode 100755 index 0000000000..b4bcac5eef --- /dev/null +++ b/scripts/skippy-workload-certify.sh @@ -0,0 +1,368 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +MODEL_CLASS="" +LANE="" +MODEL_PATH="" +MODEL_ID="" +PROJECTOR_PATH="" +WORK_DIR="" +SKIP_BUILD=0 +ORACLE_SERVER="" +ORACLE_COMPLETION="" +ORACLE_TTS="" +ORACLE_REQUIRED=0 + +usage() { + cat >&2 <<'EOF' +usage: scripts/skippy-workload-certify.sh --class CLASS --lane LANE + --model-path PATH --model-id ID --work-dir PATH [--projector-path PATH] + [--oracle-server PATH] [--oracle-completion PATH] [--oracle-tts PATH] + [--require-oracle] # fail closed unless the class-appropriate oracle is selected + [--skip-build] # skips the candidate build only for smoke-only runs +EOF +} + +while (( $# > 0 )); do + case "$1" in + --class) MODEL_CLASS="$2"; shift ;; + --lane) LANE="$2"; shift ;; + --model-path) MODEL_PATH="$2"; shift ;; + --model-id) MODEL_ID="$2"; shift ;; + --projector-path) PROJECTOR_PATH="$2"; shift ;; + --work-dir) WORK_DIR="$2"; shift ;; + --oracle-server) ORACLE_SERVER="$2"; shift ;; + --oracle-completion) ORACLE_COMPLETION="$2"; shift ;; + --oracle-tts) ORACLE_TTS="$2"; shift ;; + --require-oracle) ORACLE_REQUIRED=1 ;; + --skip-build) SKIP_BUILD=1 ;; + -h|--help) usage; exit 0 ;; + *) echo "unknown option: $1" >&2; usage; exit 1 ;; + esac + shift +done + +case "$MODEL_CLASS" in + embedding) EXPECTED_LANE="embedding-smoke" ;; + rerank) EXPECTED_LANE="rerank-smoke" ;; + encoder_decoder) EXPECTED_LANE="encoder-decoder-smoke" ;; + ocr) EXPECTED_LANE="ocr-smoke" ;; + speech_synthesis) EXPECTED_LANE="speech-synthesis-smoke" ;; + speech_recognition) EXPECTED_LANE="speech-recognition-smoke" ;; + *) echo "unsupported model class: $MODEL_CLASS" >&2; exit 1 ;; +esac + +if [[ "$LANE" != "$EXPECTED_LANE" ]]; then + echo "lane $LANE does not match class $MODEL_CLASS (expected $EXPECTED_LANE)" >&2 + exit 1 +fi +if [[ -z "$MODEL_PATH" || ! -f "$MODEL_PATH" ]]; then + echo "model path not found: $MODEL_PATH" >&2 + exit 1 +fi +if [[ -z "$MODEL_ID" || -z "$WORK_DIR" ]]; then + echo "--model-id and --work-dir are required" >&2 + exit 1 +fi +if [[ "$MODEL_CLASS" =~ ^(ocr|speech_synthesis|speech_recognition)$ ]] && [[ ! -f "$PROJECTOR_PATH" ]]; then + echo "class $MODEL_CLASS requires a projector path" >&2 + exit 1 +fi +if [[ ( -n "$ORACLE_SERVER" && ( -n "$ORACLE_COMPLETION" || -n "$ORACLE_TTS" ) ) || + ( -n "$ORACLE_COMPLETION" && -n "$ORACLE_TTS" ) ]]; then + echo "select one local-monolithic oracle executable per class" >&2 + exit 1 +fi +if (( ORACLE_REQUIRED == 1 )) && [[ -z "$ORACLE_SERVER" && -z "$ORACLE_COMPLETION" && -z "$ORACLE_TTS" ]]; then + echo "certified workload requires a class-appropriate local-monolithic oracle" >&2 + exit 1 +fi +CANDIDATE_BUILD_DIR="${LLAMA_STAGE_BUILD_DIR:-$(LLAMA_STAGE_BACKEND="${LLAMA_STAGE_BACKEND:-cpu}" LLAMA_STAGE_LINK_MODE=static "$ROOT/scripts/build-llama.sh" --print-build-dir)}" +require_pinned_cpu_oracle() { + local executable="$1" expected_name="$2" cmake_option="$3" + local build_dir stamp candidate_build_dir candidate_stamp patched_sha + if [[ ! -x "$executable" ]]; then + echo "oracle executable is not executable: $executable" >&2 + return 1 + fi + patched_sha="$(python3 "$ROOT/scripts/llama-oracle-source.py")" || return 1 + candidate_build_dir="$CANDIDATE_BUILD_DIR" + candidate_stamp="$candidate_build_dir/.mesh-llm-build-stamp" + if [[ ! -f "$candidate_stamp" ]] || + ! grep -Fxq "patched-sha=$patched_sha" "$candidate_stamp" || + ! grep -Fxq 'backend=cpu' "$candidate_stamp" || + ! grep -Fxq 'link-mode=static' "$candidate_stamp" || + ! grep -Fxq 'cmake-arg=-DGGML_METAL=OFF' "$candidate_stamp"; then + echo "candidate static ABI lacks the current pinned CPU llama.cpp build stamp" >&2 + return 1 + fi + build_dir="$(cd "$(dirname "$executable")/.." && pwd -P)" + stamp="$build_dir/.mesh-llm-build-stamp" + if [[ "$(basename "$executable")" != "$expected_name" ]] || + [[ ! -f "$stamp" ]] || + ! grep -Fxq "patched-sha=$patched_sha" "$stamp" || + ! grep -Fxq 'backend=cpu' "$stamp" || + ! grep -Fxq 'cmake-arg=-DGGML_METAL=OFF' "$stamp" || + ! grep -Fxq "$cmake_option" "$stamp"; then + echo "oracle executable lacks the current pinned CPU llama.cpp build stamp" >&2 + return 1 + fi +} +if [[ -n "$ORACLE_SERVER" ]]; then + if [[ ! "$MODEL_CLASS" =~ ^(embedding|rerank|ocr|speech_recognition)$ ]]; then + echo "class $MODEL_CLASS requires a different local-monolithic oracle executable" >&2 + exit 1 + fi + require_pinned_cpu_oracle "$ORACLE_SERVER" llama-server 'cmake-arg=-DLLAMA_BUILD_SERVER=ON' +fi +if [[ -n "$ORACLE_COMPLETION" ]]; then + if [[ "$MODEL_CLASS" != "encoder_decoder" ]]; then + echo "--oracle-completion is only valid for encoder-decoder models" >&2 + exit 1 + fi + require_pinned_cpu_oracle "$ORACLE_COMPLETION" llama-completion 'cmake-arg=-DLLAMA_BUILD_TOOLS=ON' +fi +if [[ -n "$ORACLE_TTS" ]]; then + if [[ "$MODEL_CLASS" != "speech_synthesis" ]]; then + echo "--oracle-tts is only valid for speech synthesis" >&2 + exit 1 + fi + require_pinned_cpu_oracle "$ORACLE_TTS" llama-tts 'cmake-arg=-DLLAMA_BUILD_TOOLS=ON' +fi + +mkdir -p "$WORK_DIR" +EVIDENCE_PATH="$WORK_DIR/workload-oracle-evidence.json" +COMPARISON_LOG="$WORK_DIR/workload-oracle-comparison.txt" +rm -f "$EVIDENCE_PATH" "$COMPARISON_LOG" +DIMENSIONS="$("$ROOT/scripts/plan-family-battery.py" --inspect-gguf "$MODEL_PATH")" +LAYER_END="$(jq -r '.layer_count' <<<"$DIMENSIONS")" +MODEL_SHA256="$(shasum -a 256 "$MODEL_PATH" | awk '{print $1}')" +N_GPU_LAYERS="${SKIPPY_WORKLOAD_N_GPU_LAYERS:-0}" +BACKEND="${LLAMA_STAGE_BACKEND:-cpu}" +if [[ ( -n "$ORACLE_SERVER" || -n "$ORACLE_COMPLETION" || -n "$ORACLE_TTS" ) && + ( "$N_GPU_LAYERS" != "0" || "$BACKEND" != "cpu" ) ]]; then + echo "local-monolithic comparison requires CPU-only candidate execution" >&2 + exit 1 +fi +export LLAMA_STAGE_BACKEND="$BACKEND" +if [[ -n "$ORACLE_SERVER" || -n "$ORACLE_COMPLETION" || -n "$ORACLE_TTS" ]]; then + export LLAMA_STAGE_LINK_MODE=static + export LLAMA_STAGE_BUILD_DIR="$CANDIDATE_BUILD_DIR" +fi + +# An oracle result must never be based on a stale Rust executable. Cargo tracks +# both Rust sources and the native archives, so always refresh the candidate +# when comparing against a monolithic reference, even for battery runs that +# prebuilt binaries and passed --skip-build. +if (( SKIP_BUILD == 0 )) || [[ -n "$ORACLE_SERVER" || -n "$ORACLE_COMPLETION" || -n "$ORACLE_TTS" ]]; then + LLAMA_STAGE_BUILD_DIR="$CANDIDATE_BUILD_DIR" \ + cargo build -p skippy-server +fi +if [[ -n "$ORACLE_SERVER" || -n "$ORACLE_COMPLETION" || -n "$ORACLE_TTS" ]]; then + python3 "$ROOT/scripts/check-skippy-workload-candidate.py" \ + --candidate-binary "$ROOT/target/debug/skippy-server" \ + --native-build-dir "$CANDIDATE_BUILD_DIR" +fi + +MEDIA_PATH="" +case "$MODEL_CLASS" in + ocr) MEDIA_PATH="$ROOT/ci/llama-canary/fixtures/multimodal-smoke.png" ;; + speech_recognition) MEDIA_PATH="$ROOT/ci/llama-canary/fixtures/audio-smoke.wav" ;; +esac + +env \ + SKIPPY_WORKLOAD_CLASS="$MODEL_CLASS" \ + SKIPPY_WORKLOAD_MODEL="$MODEL_PATH" \ + SKIPPY_WORKLOAD_MODEL_ID="$MODEL_ID" \ + SKIPPY_WORKLOAD_PROJECTOR="$PROJECTOR_PATH" \ + SKIPPY_WORKLOAD_MEDIA="$MEDIA_PATH" \ + SKIPPY_WORKLOAD_LAYER_END="$LAYER_END" \ + SKIPPY_WORKLOAD_CTX_SIZE="${SKIPPY_WORKLOAD_CTX_SIZE:-2048}" \ + SKIPPY_WORKLOAD_MAX_TOKENS="${SKIPPY_WORKLOAD_MAX_TOKENS:-32}" \ + SKIPPY_WORKLOAD_N_GPU_LAYERS="$N_GPU_LAYERS" \ + LLAMA_STAGE_BACKEND="$BACKEND" \ + cargo test --manifest-path "$ROOT/Cargo.toml" -p skippy-server --lib \ + frontend::tests::non_chat::real_non_chat_class_smoke_when_fixture_is_set \ + -- --nocapture --exact --test-threads=1 + +PORT="${SKIPPY_WORKLOAD_OPENAI_PORT:-19337}" +CONFIG_PATH="$WORK_DIR/stage-openai.json" +python3 - "$CONFIG_PATH" "$MODEL_ID" "$MODEL_PATH" "$MODEL_SHA256" "$LAYER_END" "$N_GPU_LAYERS" "$PROJECTOR_PATH" <<'PY' +import json +import sys + +config_path, model_id, model_path, model_sha256, layer_end, n_gpu_layers, projector_path = sys.argv[1:] +config = { + "run_id": "workload-http-smoke", + "topology_id": "workload-http-smoke-local", + "model_id": model_id, + "model_path": model_path, + "source_model_sha256": model_sha256, + "stage_id": "stage-0", + "stage_index": 0, + "layer_start": 0, + "layer_end": int(layer_end), + "ctx_size": 2048, + "lane_count": 1, + "n_batch": 2048, + "n_ubatch": 2048, + "n_gpu_layers": int(n_gpu_layers), + "selected_device": ({"backend_device": "CPU"} if int(n_gpu_layers) == 0 else None), + "kv_offload": (False if int(n_gpu_layers) == 0 else None), + "op_offload": (False if int(n_gpu_layers) == 0 else None), + "filter_tensors_on_load": False, + "native_mtp_enabled": False, + "load_mode": "runtime-slice", + "bind_addr": "127.0.0.1:0", +} +if projector_path: + config["projector_path"] = projector_path +with open(config_path, "w", encoding="utf-8") as handle: + json.dump(config, handle, indent=2) + handle.write("\n") +PY + +SERVER_LOG="$WORK_DIR/workload-openai-server.log" +LLAMA_STAGE_BACKEND="$BACKEND" \ + "$ROOT/target/debug/skippy-server" serve-openai \ + --config "$CONFIG_PATH" \ + --bind-addr "127.0.0.1:$PORT" \ + --telemetry-level off \ + >"$SERVER_LOG" 2>&1 & +SERVER_PID="$!" +ORACLE_PID="" +cleanup() { + if [[ -n "$ORACLE_PID" ]] && kill -0 "$ORACLE_PID" >/dev/null 2>&1; then + kill "$ORACLE_PID" >/dev/null 2>&1 || true + wait "$ORACLE_PID" >/dev/null 2>&1 || true + fi + if kill -0 "$SERVER_PID" >/dev/null 2>&1; then + kill "$SERVER_PID" >/dev/null 2>&1 || true + wait "$SERVER_PID" >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT + +for _ in {1..180}; do + if ! kill -0 "$SERVER_PID" >/dev/null 2>&1; then + echo "$MODEL_CLASS OpenAI server exited early" >&2 + sed -n '1,240p' "$SERVER_LOG" >&2 + exit 1 + fi + if curl -fsS --max-time 1 "http://127.0.0.1:$PORT/v1/models" 2>/dev/null \ + | jq -e --arg model "$MODEL_ID" '.data[]? | select(.id == $model)' >/dev/null 2>&1; then + break + fi + sleep 1 +done +curl -fsS --max-time 2 "http://127.0.0.1:$PORT/v1/models" \ + | jq -e --arg model "$MODEL_ID" '.data[]? | select(.id == $model)' >/dev/null +python3 "$ROOT/scripts/ci-openai-workload-smoke.py" \ + --base-url "http://127.0.0.1:$PORT/v1" \ + --model "$MODEL_ID" \ + --class "$MODEL_CLASS" \ + --media-path "$MEDIA_PATH" + +if [[ -n "$ORACLE_SERVER" ]]; then + ORACLE_PORT="${SKIPPY_WORKLOAD_ORACLE_PORT:-19338}" + if [[ ! "$ORACLE_PORT" =~ ^[0-9]+$ ]] || + (( ORACLE_PORT < 1 || ORACLE_PORT > 65535 || ORACLE_PORT == PORT )); then + echo "invalid or conflicting oracle port: $ORACLE_PORT" >&2 + exit 1 + fi + ORACLE_ARGS=( + -m "$MODEL_PATH" -a "$MODEL_ID" --host 127.0.0.1 --port "$ORACLE_PORT" + -c 2048 -b 2048 -ub 2048 -ngl 0 --parallel 1 --no-repack + ) + case "$MODEL_CLASS" in + embedding) ORACLE_ARGS+=(--embedding) ;; + rerank) ORACLE_ARGS+=(--embedding --reranking --pooling rank) ;; + ocr|speech_recognition) ORACLE_ARGS+=(--mmproj "$PROJECTOR_PATH") ;; + esac + ORACLE_LOG="$WORK_DIR/workload-monolithic-oracle-server.log" + "$ORACLE_SERVER" "${ORACLE_ARGS[@]}" >"$ORACLE_LOG" 2>&1 & + ORACLE_PID="$!" + for _ in {1..180}; do + if ! kill -0 "$ORACLE_PID" >/dev/null 2>&1; then + echo "$MODEL_CLASS monolithic oracle server exited early" >&2 + tail -80 "$ORACLE_LOG" >&2 + exit 1 + fi + if curl -fsS --max-time 1 "http://127.0.0.1:$ORACLE_PORT/v1/models" 2>/dev/null \ + | jq -e --arg model "$MODEL_ID" '.data[]? | select(.id == $model)' >/dev/null 2>&1; then + break + fi + sleep 1 + done + curl -fsS --max-time 2 "http://127.0.0.1:$ORACLE_PORT/v1/models" \ + | jq -e --arg model "$MODEL_ID" '.data[]? | select(.id == $model)' >/dev/null + if [[ "$MODEL_CLASS" =~ ^(ocr|speech_recognition)$ ]]; then + ORACLE_MEDIA_PATH="$MEDIA_PATH" + if [[ "$MODEL_CLASS" == "ocr" ]]; then + ORACLE_MEDIA_PATH="$WORK_DIR/ocr-oracle-mesh-42.png" + python3 "$ROOT/scripts/generate-ocr-oracle-fixture.py" --output "$ORACLE_MEDIA_PATH" + fi + python3 "$ROOT/scripts/skippy-ocr-asr-oracle.py" \ + --candidate-url "http://127.0.0.1:$PORT/v1" \ + --oracle-url "http://127.0.0.1:$ORACLE_PORT/v1" \ + --model "$MODEL_ID" \ + --class "$MODEL_CLASS" \ + --media-path "$ORACLE_MEDIA_PATH" | tee "$COMPARISON_LOG" + else + python3 "$ROOT/scripts/ci-workload-monolithic-oracle.py" \ + --candidate-url "http://127.0.0.1:$PORT/v1" \ + --oracle-url "http://127.0.0.1:$ORACLE_PORT/v1" \ + --model "$MODEL_ID" \ + --class "$MODEL_CLASS" | tee "$COMPARISON_LOG" + fi +fi + +if [[ -n "$ORACLE_COMPLETION" ]]; then + python3 "$ROOT/scripts/ci-workload-monolithic-oracle.py" \ + --candidate-url "http://127.0.0.1:$PORT/v1" \ + --oracle-completion "$ORACLE_COMPLETION" \ + --model-path "$MODEL_PATH" \ + --model "$MODEL_ID" \ + --class "$MODEL_CLASS" | tee "$COMPARISON_LOG" +fi + +if [[ -n "$ORACLE_TTS" ]]; then + python3 "$ROOT/scripts/skippy-tts-oracle.py" \ + --oracle-cli "$ORACLE_TTS" \ + --model-path "$MODEL_PATH" \ + --projector-path "$PROJECTOR_PATH" \ + --model "$MODEL_ID" \ + --layer-end "$LAYER_END" \ + --work-dir "$WORK_DIR" | tee "$COMPARISON_LOG" +fi + +if [[ -n "$ORACLE_SERVER" || -n "$ORACLE_COMPLETION" || -n "$ORACLE_TTS" ]]; then + ORACLE_EXECUTABLE="${ORACLE_SERVER:-${ORACLE_COMPLETION:-$ORACLE_TTS}}" + evidence_command=(python3 "$ROOT/scripts/write-workload-oracle-evidence.py" + --output "$EVIDENCE_PATH" + --comparison-log "$COMPARISON_LOG" + --class "$MODEL_CLASS" + --smoke-lane "$EXPECTED_LANE" + --model-id "$MODEL_ID" + --model-sha256 "$MODEL_SHA256" + --candidate-executable "$ROOT/target/debug/skippy-server" + --oracle-executable "$ORACLE_EXECUTABLE" + --pinned-patch-sha "$(python3 "$ROOT/scripts/llama-oracle-source.py")" + --work-dir "$WORK_DIR") + if [[ -n "$PROJECTOR_PATH" ]]; then + evidence_command+=(--projector-path "$PROJECTOR_PATH") + fi + "${evidence_command[@]}" +fi + +if [[ "$MODEL_CLASS" == "embedding" ]]; then + SDK_PYTHON="${SKIPPY_WORKLOAD_SDK_PYTHON:-python3}" + if "$SDK_PYTHON" -c 'import openai' >/dev/null 2>&1; then + "$SDK_PYTHON" "$ROOT/scripts/ci-openai-embeddings-smoke.py" \ + --base-url "http://127.0.0.1:$PORT/v1" \ + --model "$MODEL_ID" + else + echo "official openai-python SDK smoke skipped: package unavailable to $SDK_PYTHON" >&2 + fi +fi diff --git a/scripts/tests/test_check_skippy_workload_candidate.py b/scripts/tests/test_check_skippy_workload_candidate.py new file mode 100644 index 0000000000..9e5772416f --- /dev/null +++ b/scripts/tests/test_check_skippy_workload_candidate.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +CHECK = ROOT / "scripts" / "check-skippy-workload-candidate.py" + + +class CandidateBuildFreshnessTests(unittest.TestCase): + def _check(self, binary: Path, build_dir: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + sys.executable, + str(CHECK), + "--candidate-binary", + str(binary), + "--native-build-dir", + str(build_dir), + ], + capture_output=True, + text=True, + check=False, + ) + + def test_accepts_binary_linked_after_stamped_native_build(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + directory = Path(temp_dir) + stamp = directory / ".mesh-llm-build-stamp" + binary = directory / "skippy-server" + stamp.touch() + binary.touch(mode=0o755) + os.utime(stamp, ns=(1_000_000_000, 1_000_000_000)) + os.utime(binary, ns=(2_000_000_000, 2_000_000_000)) + self.assertEqual(0, self._check(binary, directory).returncode) + + def test_rejects_binary_older_than_or_equal_to_native_stamp(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + directory = Path(temp_dir) + stamp = directory / ".mesh-llm-build-stamp" + binary = directory / "skippy-server" + stamp.touch() + binary.touch(mode=0o755) + os.utime(stamp, ns=(2_000_000_000, 2_000_000_000)) + for binary_time in (1_000_000_000, 2_000_000_000): + with self.subTest(binary_time=binary_time): + os.utime(binary, ns=(binary_time, binary_time)) + result = self._check(binary, directory) + self.assertNotEqual(0, result.returncode) + self.assertIn("candidate executable predates", result.stderr) + + def test_rejects_missing_executable_or_stamp(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + directory = Path(temp_dir) + binary = directory / "skippy-server" + self.assertIn("candidate executable is missing", self._check(binary, directory).stderr) + binary.touch(mode=0o755) + self.assertIn("native build stamp is missing", self._check(binary, directory).stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_llama_native_full_replay.py b/scripts/tests/test_llama_native_full_replay.py index 77b0d6e45d..c30cbf986a 100644 --- a/scripts/tests/test_llama_native_full_replay.py +++ b/scripts/tests/test_llama_native_full_replay.py @@ -30,6 +30,10 @@ "test-skippy-recurrent-state-roundtrip", "test-skippy-verify-checkpoint-retirement", } +NON_CHAT_TARGETS = { + "test-skippy-rerank-template", + "test-skippy-sampling-suppress", +} class LlamaNativeFullReplayTests(unittest.TestCase): @@ -156,8 +160,10 @@ def test_default_build_keeps_standard_and_private_tests_disabled(self) -> None: self.assertIn("-DLLAMA_BUILD_TESTS=OFF", configure["args"]) self.assertIn("-DLLAMA_STAGE_BUILD_TESTS=OFF", configure["args"]) + self.assertIn("-DGGML_METAL=OFF", configure["args"]) self.assertTrue(PRIVATE_TARGETS.isdisjoint(build["args"])) self.assertTrue(LEGACY_TARGETS.isdisjoint(build["args"])) + self.assertTrue(NON_CHAT_TARGETS.isdisjoint(build["args"])) self.assertFalse(any(call["tool"] == "ctest" for call in trace)) def test_full_replay_builds_and_runs_only_skippy_gates(self) -> None: @@ -172,6 +178,7 @@ def test_full_replay_builds_and_runs_only_skippy_gates(self) -> None: self.assertIn("-DLLAMA_BUILD_SERVER=OFF", configure["args"]) self.assertTrue(PRIVATE_TARGETS.issubset(build["args"])) self.assertTrue(LEGACY_TARGETS.issubset(build["args"])) + self.assertTrue(NON_CHAT_TARGETS.issubset(build["args"])) self.assertNotIn("test-llama-archs", build["args"]) self.assertEqual( [fixture[:4] for fixture in fixtures], diff --git a/scripts/tests/test_llama_oracle_source.py b/scripts/tests/test_llama_oracle_source.py new file mode 100644 index 0000000000..5dfb4970d8 --- /dev/null +++ b/scripts/tests/test_llama_oracle_source.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import hashlib +import importlib.util +from pathlib import Path +import subprocess +import tempfile +import unittest + + +SCRIPT = Path(__file__).resolve().parents[1] / "llama-oracle-source.py" +SPEC = importlib.util.spec_from_file_location("llama_oracle_source", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +source = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(source) + + +class LlamaOracleSourceTests(unittest.TestCase): + def test_digest_includes_generated_series_in_prepare_order(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + patches = Path(temp_dir) + (patches / "0001-base.patch").write_bytes(b"base\n") + generated = patches / "generated" + generated.mkdir() + (generated / "series").write_text("0001-family-test.patch\n", encoding="utf-8") + (generated / "0001-family-test.patch").write_bytes(b"generated\n") + expected = hashlib.sha256() + for name, content in ( + ("0001-base.patch", b"base\n"), + ("generated/0001-family-test.patch", b"generated\n"), + ): + expected.update( + f"{name}\n{hashlib.sha256(content).hexdigest()}\n".encode("utf-8") + ) + self.assertEqual(expected.hexdigest(), source.patch_digest(patches)) + (generated / "series").write_text("0002-family-test.patch\n", encoding="utf-8") + with self.assertRaisesRegex(RuntimeError, "invalid generated patch sequence"): + source.patch_digest(patches) + + def test_prepared_checkout_rejects_patch_queue_drift(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + checkout = root / ".deps/llama.cpp" + checkout.mkdir(parents=True) + patches = root / "third_party/llama.cpp/patches" + patches.mkdir(parents=True) + patch = patches / "0001-test.patch" + patch.write_bytes(b"original\n") + upstream = root / "third_party/llama.cpp/upstream.txt" + upstream.write_text("upstream-sha\n", encoding="utf-8") + subprocess.run(["git", "init", "-q", str(checkout)], check=True) + head = subprocess.run( + ["git", "-C", str(checkout), "rev-parse", "HEAD"], + capture_output=True, text=True, check=False, + ) + if head.returncode != 0: + subprocess.run( + ["git", "-C", str(checkout), "-c", "user.name=Oracle Test", + "-c", "user.email=oracle@example.invalid", "commit", "--allow-empty", + "-qm", "initial"], + check=True, + ) + head_sha = subprocess.run( + ["git", "-C", str(checkout), "rev-parse", "HEAD"], + check=True, capture_output=True, text=True, + ).stdout.strip() + (checkout / ".mesh-llm-upstream-sha").write_text("upstream-sha\n", encoding="utf-8") + (checkout / ".mesh-llm-patch-digest").write_text( + source.patch_digest(patches) + "\n", encoding="utf-8" + ) + (checkout / ".mesh-llm-patched-sha").write_text(head_sha + "\n", encoding="utf-8") + (checkout / ".mesh-llm-prepare-schema").write_text("4\n", encoding="utf-8") + self.assertEqual(head_sha, source.prepared_patched_sha(root)) + patch.write_bytes(b"changed\n") + with self.assertRaisesRegex(RuntimeError, "does not match the current patch queue"): + source.prepared_patched_sha(root) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_llama_upstream_canary_contract.py b/scripts/tests/test_llama_upstream_canary_contract.py index 5bdb40966e..31e7ed2cbb 100644 --- a/scripts/tests/test_llama_upstream_canary_contract.py +++ b/scripts/tests/test_llama_upstream_canary_contract.py @@ -6,6 +6,7 @@ from pathlib import Path import signal import stat +import struct import subprocess import sys import tempfile @@ -536,6 +537,16 @@ def _manifest(model: dict[str, object]) -> dict[str, object]: "stage-load", ], }, + "workload-smoke": { + "status": "provisional", + "oracle": "none", + "required_lanes": ["class-specific-smoke"], + }, + "workload-oracle": { + "status": "certified", + "oracle": "local-monolithic", + "required_lanes": ["class-specific-smoke", "class-specific-oracle"], + }, }, "cadences": ["llama-bump", "manual-full", "nightly", "rotating"], }, @@ -546,6 +557,7 @@ def _manifest(model: dict[str, object]) -> dict[str, object]: def _model(revision: str = "a" * 40) -> dict[str, object]: return { "family": "test-family", + "class": "causal_generation", "profile": "full", "cadences": ["llama-bump", "manual-full"], "artifact": { @@ -651,6 +663,36 @@ def test_dry_run_reconciles_every_planned_family(self) -> None: self.assertIn("--family test-family", commands[0]) self.assertIn("--family second-family", commands[3]) + def test_supplied_plan_cannot_omit_a_manifest_selected_family(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + temp = Path(temp_dir) + first = self._model() + second = self._model() + second["family"] = "second-family" + manifest = temp / "manifest.json" + policy = self._manifest(first) + policy["models"] = [first, second] + manifest.write_text(json.dumps(policy) + "\n", encoding="utf-8") + generated = subprocess.run( + [str(ROOT / "scripts" / "plan-family-battery.py"), "--manifest", str(manifest)], + cwd=ROOT, text=True, capture_output=True, check=False, + ) + self.assertEqual(0, generated.returncode, generated.stderr) + plan = json.loads(generated.stdout) + plan["selected_models"].pop() + plan["selected_family_count"] = 1 + plan["shards"][0]["families"] = ["test-family"] + supplied = temp / "tampered-plan.json" + supplied.write_text(json.dumps(plan), encoding="utf-8") + result = subprocess.run( + [str(BATTERY), "--manifest", str(manifest), "--plan", str(supplied), + "--dry-run", "--skip-build"], + cwd=ROOT, text=True, capture_output=True, check=False, + ) + self.assertEqual(2, result.returncode) + self.assertIn("differs from the canonical manifest and selection", result.stderr) + self.assertNotIn("model-scans", result.stdout) + def test_family_filter_limits_the_resolved_dry_run(self) -> None: selected = self._dry_run("--families", "test-family") self.assertEqual(0, selected.returncode, selected.stderr) @@ -692,12 +734,15 @@ def test_mmproj_smoke_lane_runs_only_for_families_with_a_projector(self) -> None self.assertIn("SKIPPY_MM_PROJECTOR=", smokes[0]) self.assertIn("frontend::tests::multimodal", smokes[0]) self.assertIn("--test-threads=1", smokes[0]) - self.assertIn("family battery complete: 3/3", with_mmproj.stdout) + self.assertIn( + "family battery dry run complete: 3 certifications planned; no lanes executed", + with_mmproj.stdout, + ) def test_mmproj_failure_is_accounted_separately_from_core_certification(self) -> None: script = BATTERY.read_text(encoding="utf-8") smoke_body = script.split("run_mmproj_smoke() {", 1)[1].split( - "\n}\n\nrun_resolved_manifest()", 1 + "\n}\n\nrun_workload_certify()", 1 )[0] self.assertIn("MM_SMOKE_FAILURE_COUNT=0", script) @@ -732,7 +777,21 @@ def test_preflight_pins_snapshot_and_records_native_mtp_models(self) -> None: / "model.gguf" ) model.parent.mkdir(parents=True) - model.write_bytes(b"gguf-fixture") + def gguf_string(value: str) -> bytes: + encoded = value.encode("utf-8") + return struct.pack(" None: check=False, timeout=30, ) - self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) run_dir = next(artifacts.iterdir()) + environment = json.loads( + (run_dir / "preflight" / "environment.json").read_text(encoding="utf-8") + ) + self.assertFalse(environment["port_range"]["checked"]) resolved = (run_dir / "resolved-models.tsv").read_text(encoding="utf-8") self.assertIn(revision, resolved) self.assertIn("|1|1024|5|", resolved) diff --git a/scripts/tests/test_plan_family_battery.py b/scripts/tests/test_plan_family_battery.py index 61777039ba..f942e6a383 100644 --- a/scripts/tests/test_plan_family_battery.py +++ b/scripts/tests/test_plan_family_battery.py @@ -113,11 +113,27 @@ def test_checked_in_policy_resolves_all_certified_models(self) -> None: result = self._run() self.assertEqual(0, result.returncode, result.stderr) plan = json.loads(result.stdout) - self.assertEqual(77, plan["selected_family_count"]) + self.assertEqual(83, plan["selected_family_count"]) self.assertEqual( ["single-step", "chain", "state-handoff"], plan["required_certification_lanes"], ) + self.assertEqual( + { + "causal_generation", + "embedding", + "rerank", + "encoder_decoder", + "ocr", + "speech_synthesis", + "speech_recognition", + }, + {model["class"] for model in plan["selected_models"]}, + ) + self.assertEqual( + ["embedding-smoke", "embedding-oracle"], + plan["model_class_lanes"]["embedding"], + ) glm47 = next( model for model in plan["selected_models"] @@ -143,6 +159,29 @@ def test_checked_in_policy_resolves_all_certified_models(self) -> None: self.assertEqual(4, qwen4exp["execution"]["boundary_sweep_period"]) self.assertEqual(3, len(qwen4exp["artifact"]["files"])) self.assertEqual(16384, by_family["deepseek4"]["execution"]["activation_width"]) + expected_workloads = { + "nomic-bert-embedding": ("embedding", "embedding-smoke"), + "jina-bert-v2-rerank": ("rerank", "rerank-smoke"), + "t5-encoder-decoder": ( + "encoder_decoder", + "encoder-decoder-smoke", + ), + "paddleocr": ("ocr", "ocr-smoke"), + "qwen3tts": ("speech_synthesis", "speech-synthesis-smoke"), + "ultravox": ("speech_recognition", "speech-recognition-smoke"), + } + for family, (model_class, lane) in expected_workloads.items(): + with self.subTest(family=family): + model = by_family[family] + self.assertEqual(model_class, model["class"]) + self.assertEqual([lane, lane.replace("-smoke", "-oracle")], model["certification_lanes"]) + self.assertEqual("workload-oracle", model["profile"]) + self.assertEqual("certified", model["certification_status"]) + self.assertEqual("local-monolithic", model["oracle"]) + self.assertEqual(["manual-full"], model["cadences"]) + self.assertEqual("disabled", model["execution"]["speculative_policy"]) + self.assertEqual(0, model["execution"]["boundary_sweep_period"]) + self.assertEqual(0, model["execution"]["mtp_layers"]) def test_nightly_cadence_selects_cache_mechanism_sentinels(self) -> None: result = self._run(MANIFEST, "--cadence", "nightly") @@ -184,7 +223,16 @@ def test_mmproj_artifacts_resolve_and_cover_the_vision_families(self) -> None: if model.get("mmproj_artifact") is not None } self.assertEqual( - {"gemma4", "lfm2-vl", "muse-glimmer", "qwen2-vl", "qwen3-vl"}, + { + "gemma4", + "lfm2-vl", + "muse-glimmer", + "qwen2-vl", + "qwen3-vl", + "paddleocr", + "qwen3tts", + "ultravox", + }, set(with_mmproj), ) for family, mmproj in with_mmproj.items(): @@ -194,9 +242,7 @@ def test_mmproj_artifacts_resolve_and_cover_the_vision_families(self) -> None: self.assertEqual( set(mmproj["file_integrity"]), set(mmproj["files"]) ) - self.assertRegex( - mmproj["files"][0], r"^mmproj" - ) + self.assertIn("mmproj", mmproj["files"][0].lower()) def test_certified_model_requires_an_explicit_activation_width(self) -> None: manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) @@ -232,6 +278,76 @@ def test_certified_profile_cannot_add_or_reorder_core_lanes(self) -> None: self.assertEqual(2, result.returncode) self.assertIn("must require exactly the three core lanes", result.stderr) + def test_workload_smoke_profile_cannot_claim_an_oracle(self) -> None: + manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) + manifest["policy"]["profiles"]["workload-smoke"]["oracle"] = "local-monolithic" + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "manifest.json" + path.write_text(json.dumps(manifest), encoding="utf-8") + result = self._run(path) + self.assertEqual(2, result.returncode) + self.assertIn("workload-smoke must remain provisional and oracle-free", result.stderr) + + def test_workload_oracle_profile_cannot_drop_the_oracle_lane(self) -> None: + manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) + manifest["policy"]["profiles"]["workload-oracle"]["required_lanes"] = ["class-specific-smoke"] + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "manifest.json" + path.write_text(json.dumps(manifest), encoding="utf-8") + result = self._run(path) + self.assertEqual(2, result.returncode) + self.assertIn("workload-oracle requires certified local-monolithic", result.stderr) + + def test_certified_workload_requires_fixture_and_comparison_evidence(self) -> None: + manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) + model = next(row for row in manifest["models"] if row["profile"] == "workload-oracle") + del model["evidence"] + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "manifest.json" + path.write_text(json.dumps(manifest), encoding="utf-8") + result = self._run(path) + self.assertEqual(2, result.returncode) + self.assertIn("evidence must be an object", result.stderr) + + def test_supplied_plan_rejects_tampered_selected_model_rows(self) -> None: + generated = self._run(MANIFEST, "--cadence", "manual-full", "--shard-count", "2") + self.assertEqual(0, generated.returncode, generated.stderr) + plan = json.loads(generated.stdout) + family = plan["selected_models"].pop()["family"] + plan["selected_family_count"] -= 1 + for shard in plan["shards"]: + if family in shard["families"]: + shard["families"].remove(family) + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "tampered-plan.json" + path.write_text(json.dumps(plan), encoding="utf-8") + result = self._run(MANIFEST, "--verify-plan", str(path)) + self.assertEqual(2, result.returncode) + self.assertIn("differs from the canonical manifest and selection", result.stderr) + + def test_supplied_plan_rejects_inflated_oracle_status(self) -> None: + generated = self._run(MANIFEST, "--families", "nomic-bert-embedding") + self.assertEqual(0, generated.returncode, generated.stderr) + plan = json.loads(generated.stdout) + plan["selected_models"][0]["oracle"] = "none" + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "tampered-plan.json" + path.write_text(json.dumps(plan), encoding="utf-8") + result = self._run(MANIFEST, "--verify-plan", str(path)) + self.assertEqual(2, result.returncode) + self.assertIn("differs from the canonical manifest and selection", result.stderr) + + def test_non_chat_model_cannot_claim_the_certified_full_profile(self) -> None: + manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) + model = next(row for row in manifest["models"] if row["class"] == "embedding") + model["profile"] = "full" + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "manifest.json" + path.write_text(json.dumps(manifest), encoding="utf-8") + result = self._run(path) + self.assertEqual(2, result.returncode) + self.assertIn("requires a class-specific workload profile", result.stderr) + def test_duplicate_family_is_rejected(self) -> None: manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) manifest["models"].append(copy.deepcopy(manifest["models"][0])) @@ -242,6 +358,93 @@ def test_duplicate_family_is_rejected(self) -> None: self.assertEqual(2, result.returncode) self.assertIn("duplicate family", result.stderr) + def test_model_class_is_required_and_selects_class_specific_lanes(self) -> None: + source = json.loads(MANIFEST.read_text(encoding="utf-8")) + model = copy.deepcopy(source["models"][0]) + source["models"] = [model] + del model["class"] + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "manifest.json" + path.write_text(json.dumps(source), encoding="utf-8") + missing = self._run(path) + model["class"] = "embedding" + model["profile"] = "workload-smoke" + model["execution"]["speculative_policy"] = "disabled" + path.write_text(json.dumps(source), encoding="utf-8") + embedding = self._run(path) + + self.assertEqual(2, missing.returncode) + self.assertIn("models[0].class must be", missing.stderr) + self.assertEqual(0, embedding.returncode, embedding.stderr) + selected = json.loads(embedding.stdout)["selected_models"][0] + self.assertEqual("embedding", selected["class"]) + self.assertEqual(["embedding-smoke"], selected["certification_lanes"]) + + def test_projector_classes_require_an_explicit_projector_artifact(self) -> None: + source = json.loads(MANIFEST.read_text(encoding="utf-8")) + model = copy.deepcopy(source["models"][0]) + source["models"] = [model] + model["class"] = "speech_recognition" + model["profile"] = "workload-smoke" + model["execution"]["speculative_policy"] = "disabled" + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "manifest.json" + path.write_text(json.dumps(source), encoding="utf-8") + result = self._run(path) + + self.assertEqual(2, result.returncode) + self.assertIn("requires an mmproj_artifact", result.stderr) + + def test_non_causal_classes_reject_split_and_speculative_policy(self) -> None: + source = json.loads(MANIFEST.read_text(encoding="utf-8")) + model = copy.deepcopy(source["models"][0]) + source["models"] = [model] + model["class"] = "embedding" + model["profile"] = "workload-smoke" + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "manifest.json" + path.write_text(json.dumps(source), encoding="utf-8") + speculative = self._run(path) + model["execution"]["speculative_policy"] = "disabled" + model["execution"]["boundary_sweep_period"] = 1 + path.write_text(json.dumps(source), encoding="utf-8") + split = self._run(path) + + self.assertEqual(2, speculative.returncode) + self.assertIn("must disable speculative decoding", speculative.stderr) + self.assertEqual(2, split.returncode) + self.assertIn("must not request split or MTP certification", split.stderr) + + def test_inspect_gguf_reports_canonical_dimensions_without_a_manifest(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "fixture.gguf" + self._write_gguf(path, 7, 1536) + result = subprocess.run( + [str(PLANNER), "--inspect-gguf", str(path)], + cwd=ROOT, + text=True, + capture_output=True, + check=False, + ) + + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual( + {"activation_width": 1536, "layer_count": 7}, + json.loads(result.stdout), + ) + + def test_unknown_model_class_is_rejected(self) -> None: + source = json.loads(MANIFEST.read_text(encoding="utf-8")) + source["models"] = [copy.deepcopy(source["models"][0])] + source["models"][0]["class"] = "guessed-from-name" + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "manifest.json" + path.write_text(json.dumps(source), encoding="utf-8") + result = self._run(path) + + self.assertEqual(2, result.returncode) + self.assertIn("class must be one of", result.stderr) + def test_cache_gate_requires_every_exact_revision_file(self) -> None: source = json.loads(MANIFEST.read_text(encoding="utf-8")) source["models"] = [copy.deepcopy(source["models"][0])] @@ -521,8 +724,8 @@ def test_shards_are_deterministic_and_preserve_every_family_once(self) -> None: families = [ family for shard in plan["shards"] for family in shard["families"] ] - self.assertEqual(77, len(families)) - self.assertEqual(77, len(set(families))) + self.assertEqual(83, len(families)) + self.assertEqual(83, len(set(families))) self.assertEqual(4, len(plan["github_matrix"]["include"])) diff --git a/scripts/tests/test_skippy_ocr_asr_oracle.py b/scripts/tests/test_skippy_ocr_asr_oracle.py new file mode 100644 index 0000000000..3b431b2dee --- /dev/null +++ b/scripts/tests/test_skippy_ocr_asr_oracle.py @@ -0,0 +1,124 @@ +"""Regression tests for deterministic OCR media and multimodal parity checks.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +import struct +import unittest +from unittest.mock import patch +import zlib + + +ROOT = Path(__file__).resolve().parents[2] + + +def import_script(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +fixture = import_script("ocr_oracle_fixture", ROOT / "scripts" / "generate-ocr-oracle-fixture.py") +oracle = import_script("skippy_ocr_asr_oracle", ROOT / "scripts" / "skippy-ocr-asr-oracle.py") + + +class OcrFixtureTests(unittest.TestCase): + def test_png_is_deterministic_and_text_bearing(self): + png = fixture.png_bytes() + self.assertEqual(png, fixture.png_bytes()) + self.assertTrue(png.startswith(b"\x89PNG\r\n\x1a\n")) + self.assertEqual(fixture.TEXT, "MESH 42") + size = struct.unpack(">II", png[16:24]) + self.assertEqual(size, (fixture.WIDTH, fixture.HEIGHT)) + + image_data_start = png.index(b"IDAT") + 4 + compressed_length = struct.unpack(">I", png[image_data_start - 8:image_data_start - 4])[0] + raw = zlib.decompress(png[image_data_start:image_data_start + compressed_length]) + self.assertEqual(len(raw), fixture.HEIGHT * (1 + fixture.WIDTH * 3)) + self.assertIn(b"\x00\x00\x00", raw) + self.assertIn(b"\xff\xff\xff", raw) + + +class MultimodalOracleTests(unittest.TestCase): + def test_ocr_requires_both_parity_and_known_text(self): + self.assertIn("mesh 42", oracle.compare_text("MESH 42", "Mesh 42.", "MESH 42")) + with self.assertRaisesRegex(RuntimeError, "differs from monolithic"): + oracle.compare_text("MESH 42", "MESH 43", "MESH 42") + with self.assertRaisesRegex(RuntimeError, "misses independently known"): + oracle.compare_text("unrelated", "Unrelated.", "MESH 42") + + def test_asr_unlabeled_fixture_does_not_claim_accuracy(self): + detail = oracle.compare_text("The mesh is ready.", "the mesh is ready", None) + self.assertIn("no accuracy claim", detail) + with self.assertRaisesRegex(RuntimeError, "returned empty text"): + oracle.compare_text("!", "?", None) + + def test_asr_normalizes_only_known_decorative_prefixes(self): + detail = oracle.compare_text( + 'The text is: "The mesh is ready"', + 'The audio is: "The mesh is ready"', + None, + transcript=True, + ) + self.assertIn("identical normalized text", detail) + with self.assertRaisesRegex(RuntimeError, "differs from monolithic"): + oracle.compare_text("The text is: mesh ready", "The audio is: mesh not ready", + None, transcript=True) + with self.assertRaisesRegex(RuntimeError, "differs from monolithic"): + oracle.compare_text("Transcribed text: mesh ready", "The audio is: mesh ready", + None, transcript=True) + + def test_asr_matching_refusals_do_not_pass_as_transcripts(self): + with self.assertRaisesRegex(RuntimeError, "not a transcript"): + oracle.compare_text("I can't fulfill this request.", + "I can't fulfill this request.", None, transcript=True) + with self.assertRaisesRegex(RuntimeError, "not a transcript"): + oracle.compare_text("I can help you with transcribing audio to text.", + "I can help you with transcribing audio to text.", + None, transcript=True) + + def test_ocr_sends_same_request_to_both_servers(self): + reply = {"choices": [{"message": {"content": "MESH 42"}}]} + with patch.object(oracle, "request_json", side_effect=[reply, reply]) as request: + oracle.compare_ocr("http://candidate/v1", "http://reference/v1", "ocr", b"png", "MESH 42") + self.assertEqual(request.call_count, 2) + candidate_call, reference_call = request.call_args_list + self.assertEqual(candidate_call.args[1:], reference_call.args[1:]) + self.assertEqual(candidate_call.args[1], "/chat/completions") + + def test_asr_aligns_reference_chat_prompt_with_candidate_audio_route(self): + with ( + patch.object(oracle, "request_multipart", return_value={"text": "The mesh is ready."}) as candidate, + patch.object(oracle, "request_json", return_value={ + "choices": [{"message": {"content": "The mesh is ready."}}] + }) as reference, + ): + oracle.compare_asr("http://candidate/v1", "http://reference/v1", "asr", b"wav", None) + self.assertEqual(candidate.call_args.args, + ("http://candidate/v1", "/audio/transcriptions", "asr", b"wav")) + self.assertEqual(reference.call_args.args[:2], + ("http://reference/v1", "/chat/completions")) + payload = reference.call_args.args[2] + parts = payload["messages"][0]["content"] + self.assertEqual(parts[0]["text"], oracle.ASR_PROMPT) + self.assertEqual(parts[1]["input_audio"]["data"], "d2F2") + + def test_asr_multipart_contains_deterministic_fields_and_audio(self): + with patch.object(oracle, "response_json", return_value={"text": "ok"}) as response: + oracle.request_multipart("http://localhost/v1/", "/audio/transcriptions", "asr", b"RIFF\x00audio") + request = response.call_args.args[0] + self.assertEqual(request.full_url, "http://localhost/v1/audio/transcriptions") + body = request.data + self.assertIn(b'name="model"\r\n\r\nasr\r\n', body) + self.assertIn(b'name="response_format"\r\n\r\njson\r\n', body) + self.assertIn(b'name="temperature"\r\n\r\n0\r\n', body) + self.assertIn(b"RIFF\x00audio", body) + with self.assertRaisesRegex(RuntimeError, "collides with multipart boundary"): + oracle.request_multipart("http://localhost/v1", "/audio/transcriptions", "asr", + oracle.BOUNDARY.encode("ascii")) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_skippy_static_link.py b/scripts/tests/test_skippy_static_link.py new file mode 100644 index 0000000000..91dfb3bf55 --- /dev/null +++ b/scripts/tests/test_skippy_static_link.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +BUILD_SCRIPT = ROOT / "crates/skippy-ffi/build.rs" +CORE_ARCHIVES = ( + "src/libllama.a", + "common/libllama-common.a", + "common/libllama-common-base.a", + "ggml/src/libggml.a", + "ggml/src/libggml-base.a", + "ggml/src/libggml-cpu.a", +) +OPTIONAL_ARCHIVES = ( + "ggml/src/ggml-blas/libggml-blas.a", + "ggml/src/ggml-cuda/libggml-cuda.a", + "ggml/src/ggml-hip/libggml-hip.a", + "ggml/src/ggml-vulkan/libggml-vulkan.a", + "ggml/src/ggml-metal/libggml-metal.a", +) + + +class SkippyStaticLinkTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.binary_dir = tempfile.TemporaryDirectory() + cls.addClassCleanup(cls.binary_dir.cleanup) + cls.binary = Path(cls.binary_dir.name) / "build-script" + result = subprocess.run( + ["rustc", "--edition=2024", str(BUILD_SCRIPT), "-o", str(cls.binary)], + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + if result.returncode: + raise RuntimeError(f"build script fixture failed: {result.stderr}") + + def _run(self, backend: str, flags: dict[str, str]) -> subprocess.CompletedProcess[str]: + fixture = tempfile.TemporaryDirectory() + self.addCleanup(fixture.cleanup) + build_dir = Path(fixture.name) / "native" + for relative in (*CORE_ARCHIVES, *OPTIONAL_ARCHIVES): + archive = build_dir / relative + archive.parent.mkdir(parents=True, exist_ok=True) + archive.touch() + (build_dir / "CMakeCache.txt").write_text( + "".join(f"{key}:BOOL={value}\n" for key, value in flags.items()), + encoding="utf-8", + ) + env = { + key: value for key, value in os.environ.items() + if not key.startswith(("LLAMA_STAGE_", "SKIPPY_LLAMA_", "CARGO_FEATURE_")) + } + env.update({ + "CARGO_MANIFEST_DIR": str(ROOT / "crates/skippy-ffi"), + "TARGET": "aarch64-apple-darwin", + "LLAMA_STAGE_BACKEND": backend, + "LLAMA_STAGE_BUILD_DIR": str(build_dir), + "LLAMA_STAGE_LINK_MODE": "static", + "SKIPPY_LLAMA_AUTO_BUILD": "0", + }) + return subprocess.run( + [str(self.binary)], cwd=ROOT, env=env, + capture_output=True, text=True, check=False, + ) + + def test_cpu_ignores_stale_gpu_and_blas_archives(self) -> None: + result = self._run("cpu", { + "GGML_BLAS": "OFF", "GGML_CUDA": "OFF", "GGML_HIP": "OFF", + "GGML_VULKAN": "OFF", "GGML_METAL": "OFF", + }) + self.assertEqual(0, result.returncode, result.stderr) + for library in ("ggml-blas", "ggml-cuda", "ggml-hip", "ggml-vulkan", "ggml-metal"): + self.assertNotIn(f"cargo:rustc-link-lib=static={library}", result.stdout) + for framework in ("Foundation", "Metal", "MetalKit"): + self.assertNotIn(f"cargo:rustc-link-lib=framework={framework}", result.stdout) + + def test_active_metal_backend_links_only_cache_enabled_archive(self) -> None: + result = self._run("metal", {"GGML_METAL": "ON", "GGML_BLAS": "ON"}) + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn("cargo:rustc-link-lib=static=ggml-metal", result.stdout) + self.assertIn("cargo:rustc-link-lib=framework=Metal", result.stdout) + self.assertIn("cargo:rustc-link-lib=static=ggml-blas", result.stdout) + self.assertNotIn("cargo:rustc-link-lib=static=ggml-cuda", result.stdout) + + def test_backend_cache_mismatch_fails_closed_despite_stale_archive(self) -> None: + result = self._run("metal", {"GGML_METAL": "OFF"}) + self.assertNotEqual(0, result.returncode) + self.assertIn("selected backend requires GGML_METAL=ON", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_skippy_tts_oracle.py b/scripts/tests/test_skippy_tts_oracle.py new file mode 100644 index 0000000000..1fdfb53031 --- /dev/null +++ b/scripts/tests/test_skippy_tts_oracle.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +import argparse +from array import array +import importlib.util +import os +from pathlib import Path +import sys +import tempfile +import unittest +from unittest import mock +import wave + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts" / "skippy-tts-oracle.py" +SPEC = importlib.util.spec_from_file_location("skippy_tts_oracle", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +oracle = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(oracle) + + +def write_wav(path: Path, samples: list[int], *, rate: int = 8000) -> None: + pcm = array("h", samples) + if sys.byteorder != "little": + pcm.byteswap() + with wave.open(str(path), "wb") as output: + output.setnchannels(1) + output.setsampwidth(2) + output.setframerate(rate) + output.writeframes(pcm.tobytes()) + + +class TtsOracleTests(unittest.TestCase): + def test_oracle_invocation_matches_candidate_no_repack_context(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + test_root = Path(temp_dir) + model = test_root / "model.gguf" + projector = test_root / "projector.gguf" + model.write_bytes(b"fixture") + projector.write_bytes(b"fixture") + work_dir = test_root / "evidence" + commands: list[list[str]] = [] + + def fake_run_logged(command: list[str], _log_path: Path, **_kwargs: object) -> None: + commands.append(command) + name = "tts-candidate.wav" if command[0] == "cargo" else "tts-monolithic-oracle.wav" + write_wav(work_dir / name, [1000, -1000] * 800) + + args = argparse.Namespace( + oracle_cli=str(test_root / "bin" / "llama-tts"), + model_path=str(model), + projector_path=str(projector), + model="qwen3-tts", + layer_end=28, + work_dir=str(work_dir), + ) + with mock.patch.dict(os.environ, {"LLAMA_STAGE_BUILD_DIR": str(test_root / "abi")}): + with mock.patch.object(oracle, "require_pinned_cpu_oracle", return_value="pinned-sha"): + with mock.patch.object(oracle, "require_candidate_cpu_static_build"): + with mock.patch.object(oracle, "run_logged", side_effect=fake_run_logged): + result = oracle.run_oracle(args) + + self.assertEqual("pass", result["status"]) + self.assertEqual(2, len(commands)) + self.assertIn("--no-repack", commands[1]) + self.assertEqual("0", commands[1][commands[1].index("--min-p") + 1]) + self.assertEqual("0", commands[1][commands[1].index("-ngl") + 1]) + + def test_identical_pcm_passes_with_zero_error(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + candidate = Path(temp_dir) / "candidate.wav" + reference = Path(temp_dir) / "reference.wav" + samples = [1000 if index % 2 else -1000 for index in range(1600)] + write_wav(candidate, samples) + write_wav(reference, samples) + metrics = oracle.compare_wavs(candidate, reference) + self.assertEqual(0.0, metrics["relative_rms_error"]) + self.assertEqual(1.0, metrics["waveform_cosine"]) + self.assertEqual(1600, metrics["sample_count"]) + + def test_small_pcm_rounding_difference_is_tolerated(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + candidate = Path(temp_dir) / "candidate.wav" + reference = Path(temp_dir) / "reference.wav" + write_wav(candidate, [1001, -999] * 800) + write_wav(reference, [1000, -1000] * 800) + metrics = oracle.compare_wavs(candidate, reference) + self.assertLess(metrics["relative_rms_error"], oracle.MAX_RELATIVE_RMS_ERROR) + + def test_wrong_gain_and_phase_are_rejected(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + candidate = Path(temp_dir) / "candidate.wav" + reference = Path(temp_dir) / "reference.wav" + write_wav(reference, [1000, -1000] * 800) + for changed in ([1060, -1060] * 800, [-1000, 1000] * 800): + with self.subTest(changed=changed[:2]): + write_wav(candidate, changed) + with self.assertRaisesRegex(RuntimeError, "differs from monolithic oracle"): + oracle.compare_wavs(candidate, reference) + + def test_silent_and_mismatched_wav_are_rejected(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + candidate = Path(temp_dir) / "candidate.wav" + reference = Path(temp_dir) / "reference.wav" + write_wav(reference, [1000, -1000] * 800) + write_wav(candidate, [0] * 1600) + with self.assertRaisesRegex(RuntimeError, "silent"): + oracle.compare_wavs(candidate, reference) + write_wav(candidate, [1000, -1000] * 799) + with self.assertRaisesRegex(RuntimeError, "sample count differs"): + oracle.compare_wavs(candidate, reference) + write_wav(candidate, [1000, -1000] * 800, rate=16000) + with self.assertRaisesRegex(RuntimeError, "sample rate"): + oracle.compare_wavs(candidate, reference) + + def test_oracle_requires_current_pinned_cpu_stamp(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + test_root = Path(temp_dir) + binary = test_root / "build" / "bin" / "llama-tts" + binary.parent.mkdir(parents=True) + binary.write_bytes(b"binary") + binary.chmod(0o755) + patched_sha = test_root / ".deps" / "llama.cpp" / ".mesh-llm-patched-sha" + patched_sha.parent.mkdir(parents=True) + patched_sha.write_text("pinned-sha\n", encoding="utf-8") + stamp = test_root / "build" / ".mesh-llm-build-stamp" + with mock.patch.object(oracle, "ROOT", test_root): + with self.assertRaisesRegex(RuntimeError, "pinned build stamp"): + oracle.require_pinned_cpu_oracle(binary) + stamp.write_text( + "patched-sha=pinned-sha\nbackend=cpu\nlink-mode=static\n" + "ggml-native=OFF\ncmake-arg=-DGGML_NATIVE=OFF\n" + "cmake-arg=-DGGML_METAL=OFF\n" + "cmake-arg=-DLLAMA_BUILD_TOOLS=ON\n", + encoding="utf-8", + ) + oracle.require_pinned_cpu_oracle(binary) + stamp.write_text( + "patched-sha=pinned-sha\nbackend=cpu\nlink-mode=static\n" + "ggml-native=ON\ncmake-arg=-DGGML_NATIVE=ON\n" + "cmake-arg=-DGGML_METAL=OFF\n" + "cmake-arg=-DLLAMA_BUILD_TOOLS=ON\n", + encoding="utf-8", + ) + with self.assertRaisesRegex(RuntimeError, "pinned CPU build stamp"): + oracle.require_pinned_cpu_oracle(binary) + stamp.write_text("patched-sha=pinned-sha\nbackend=metal\n", encoding="utf-8") + with self.assertRaisesRegex(RuntimeError, "pinned CPU build stamp"): + oracle.require_pinned_cpu_oracle(binary) + + def test_candidate_requires_same_pinned_cpu_static_build(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + build_dir = Path(temp_dir) + stamp = build_dir / ".mesh-llm-build-stamp" + with self.assertRaisesRegex(RuntimeError, "lacks a pinned static CPU build stamp"): + oracle.require_candidate_cpu_static_build(build_dir, "pinned-sha") + stamp.write_text( + "patched-sha=old-sha\nbackend=cpu\nlink-mode=static\n" + "ggml-native=OFF\ncmake-arg=-DGGML_NATIVE=OFF\n" + "cmake-arg=-DGGML_METAL=OFF\n", + encoding="utf-8", + ) + with self.assertRaisesRegex(RuntimeError, "does not match"): + oracle.require_candidate_cpu_static_build(build_dir, "pinned-sha") + stamp.write_text( + "patched-sha=pinned-sha\nbackend=cpu\nlink-mode=static\n" + "ggml-native=OFF\ncmake-arg=-DGGML_NATIVE=OFF\n" + "cmake-arg=-DGGML_METAL=OFF\n", + encoding="utf-8", + ) + oracle.require_candidate_cpu_static_build(build_dir, "pinned-sha") + stamp.write_text( + "patched-sha=pinned-sha\nbackend=cpu\nlink-mode=static\n" + "ggml-native=ON\ncmake-arg=-DGGML_NATIVE=ON\n" + "cmake-arg=-DGGML_METAL=OFF\n", + encoding="utf-8", + ) + with self.assertRaisesRegex(RuntimeError, "does not match"): + oracle.require_candidate_cpu_static_build(build_dir, "pinned-sha") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_skippy_workload_certify.py b/scripts/tests/test_skippy_workload_certify.py new file mode 100644 index 0000000000..9c3f9d232a --- /dev/null +++ b/scripts/tests/test_skippy_workload_certify.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +from pathlib import Path +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +RUNNER = ROOT / "scripts" / "skippy-workload-certify.sh" + + +class WorkloadCertifyContractTests(unittest.TestCase): + def _run(self, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [str(RUNNER), *args], + cwd=ROOT, + text=True, + capture_output=True, + check=False, + ) + + def test_help_documents_the_typed_certification_inputs(self) -> None: + result = self._run("--help") + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn("--class CLASS", result.stderr) + self.assertIn("--lane LANE", result.stderr) + self.assertIn("--projector-path PATH", result.stderr) + self.assertIn("--oracle-server PATH", result.stderr) + self.assertIn("--oracle-completion PATH", result.stderr) + self.assertIn("--oracle-tts PATH", result.stderr) + self.assertIn("--require-oracle", result.stderr) + + def test_certified_mode_rejects_missing_oracle_before_build(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + model = Path(temp_dir) / "model.gguf" + model.touch() + result = self._run( + "--class", "embedding", "--lane", "embedding-smoke", + "--model-path", str(model), "--model-id", "fixture", + "--work-dir", temp_dir, "--require-oracle", "--skip-build", + ) + self.assertEqual(1, result.returncode) + self.assertIn("certified workload requires a class-appropriate", result.stderr) + + def test_unknown_class_fails_before_model_execution(self) -> None: + result = self._run("--class", "guessed", "--lane", "guessed-equivalence") + self.assertEqual(1, result.returncode) + self.assertIn("unsupported model class: guessed", result.stderr) + + def test_lane_must_match_the_selected_class(self) -> None: + result = self._run( + "--class", + "embedding", + "--lane", + "rerank-smoke", + ) + self.assertEqual(1, result.returncode) + self.assertIn("expected embedding-smoke", result.stderr) + + def test_projector_classes_fail_closed_without_a_projector(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + model = Path(temp_dir) / "model.gguf" + model.touch() + for model_class, lane in ( + ("ocr", "ocr-smoke"), + ("speech_synthesis", "speech-synthesis-smoke"), + ("speech_recognition", "speech-recognition-smoke"), + ): + with self.subTest(model_class=model_class): + result = self._run( + "--class", + model_class, + "--lane", + lane, + "--model-path", + str(model), + "--model-id", + "fixture", + "--work-dir", + temp_dir, + "--skip-build", + ) + self.assertEqual(1, result.returncode) + self.assertIn("requires a projector path", result.stderr) + + def test_oracle_requires_a_pinned_cpu_server_binary(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + model = Path(temp_dir) / "model.gguf" + model.touch() + result = self._run( + "--class", "embedding", "--lane", "embedding-smoke", + "--model-path", str(model), "--model-id", "fixture", + "--work-dir", temp_dir, "--oracle-server", str(model), + "--skip-build", + ) + self.assertEqual(1, result.returncode) + self.assertIn("oracle executable is not executable", result.stderr) + + def test_speech_synthesis_rejects_server_oracle(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + model = Path(temp_dir) / "model.gguf" + projector = Path(temp_dir) / "projector.gguf" + model.touch() + projector.touch() + result = self._run( + "--class", "speech_synthesis", "--lane", "speech-synthesis-smoke", + "--model-path", str(model), "--projector-path", str(projector), + "--model-id", "fixture", "--work-dir", temp_dir, + "--oracle-server", str(model), "--skip-build", + ) + self.assertEqual(1, result.returncode) + self.assertIn("requires a different local-monolithic oracle", result.stderr) + + def test_other_classes_reject_tts_oracle(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + model = Path(temp_dir) / "model.gguf" + model.touch() + result = self._run( + "--class", "embedding", "--lane", "embedding-smoke", + "--model-path", str(model), "--model-id", "fixture", + "--work-dir", temp_dir, "--oracle-tts", str(model), + "--skip-build", + ) + self.assertEqual(1, result.returncode) + self.assertIn("only valid for speech synthesis", result.stderr) + + def test_encoder_decoder_requires_direct_completion_oracle(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + model = Path(temp_dir) / "model.gguf" + model.touch() + result = self._run( + "--class", "encoder_decoder", "--lane", "encoder-decoder-smoke", + "--model-path", str(model), "--model-id", "fixture", + "--work-dir", temp_dir, "--oracle-server", str(model), + "--skip-build", + ) + self.assertEqual(1, result.returncode) + self.assertIn("requires a different local-monolithic oracle", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_static_abi_artifacts.py b/scripts/tests/test_static_abi_artifacts.py index a7bf5a0696..68b4b1c8fc 100644 --- a/scripts/tests/test_static_abi_artifacts.py +++ b/scripts/tests/test_static_abi_artifacts.py @@ -123,7 +123,9 @@ def test_skippy_ffi_uses_the_native_build_scripts_canonical_directory(self) -> N for backend in ("cpu", "metal", "cuda", "rocm"): environment = os.environ.copy() + environment.pop("LLAMA_STAGE_BUILD_DIR", None) environment["LLAMA_STAGE_BACKEND"] = backend + environment["LLAMA_STAGE_LINK_MODE"] = "static" result = subprocess.run( ["bash", str(ROOT / "scripts" / "build-llama.sh"), "--print-build-dir"], cwd=ROOT, diff --git a/scripts/tests/test_verify_workload_oracle_evidence.py b/scripts/tests/test_verify_workload_oracle_evidence.py new file mode 100644 index 0000000000..7c8a1aebaa --- /dev/null +++ b/scripts/tests/test_verify_workload_oracle_evidence.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +VERIFIER = ROOT / "scripts" / "verify-workload-oracle-evidence.py" +WRITER = ROOT / "scripts" / "write-workload-oracle-evidence.py" + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +class WorkloadOracleEvidenceTests(unittest.TestCase): + def setUp(self) -> None: + self.temp_dir = tempfile.TemporaryDirectory() + self.addCleanup(self.temp_dir.cleanup) + root = Path(self.temp_dir.name) + self.model = root / "model.gguf" + self.model.write_bytes(b"pinned-model") + self.candidate = root / "skippy-server" + self.candidate.write_bytes(b"candidate") + self.oracle = root / "llama-server" + self.oracle.write_bytes(b"monolithic") + self.evidence = root / "workload-oracle-evidence.json" + self.body = { + "status": "pass", + "class": "embedding", + "smoke_lane": "embedding-smoke", + "oracle_lane": "embedding-oracle", + "model_id": "fixture", + "model_sha256": sha256(self.model), + "projector_sha256": None, + "candidate_executable_sha256": sha256(self.candidate), + "oracle_executable": "llama-server", + "oracle_executable_sha256": sha256(self.oracle), + "pinned_patch_sha": "a" * 40, + "comparison": "embedding local-monolithic oracle passed: max_abs_delta=0, min_cosine=1", + } + + def run_verifier(self) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + "python3", str(VERIFIER), "--evidence", str(self.evidence), + "--class", "embedding", "--smoke-lane", "embedding-smoke", + "--oracle-lane", "embedding-oracle", "--model-id", "fixture", + "--model-path", str(self.model), + "--candidate-executable", str(self.candidate), + "--oracle-executable", str(self.oracle), + "--pinned-patch-sha", "a" * 40, + ], + cwd=ROOT, text=True, capture_output=True, check=False, + ) + + def run_writer(self, comparison: str) -> subprocess.CompletedProcess[str]: + comparison_log = Path(self.temp_dir.name) / "comparison.txt" + comparison_log.write_text(comparison + "\n", encoding="utf-8") + return subprocess.run( + [ + "python3", str(WRITER), "--output", str(self.evidence), + "--comparison-log", str(comparison_log), "--class", "embedding", + "--smoke-lane", "embedding-smoke", "--model-id", "fixture", + "--model-sha256", sha256(self.model), + "--candidate-executable", str(self.candidate), + "--oracle-executable", str(self.oracle), + "--pinned-patch-sha", "a" * 40, + "--work-dir", self.temp_dir.name, + ], + cwd=ROOT, text=True, capture_output=True, check=False, + ) + + def test_comparator_pass_writes_verifiable_identity_bound_evidence(self) -> None: + written = self.run_writer(self.body["comparison"]) + self.assertEqual(0, written.returncode, written.stderr) + self.assertEqual(0, self.run_verifier().returncode) + + def test_smoke_only_log_never_writes_oracle_evidence(self) -> None: + written = self.run_writer("embedding OpenAI HTTP smoke passed") + self.assertEqual(1, written.returncode) + self.assertFalse(self.evidence.exists()) + + def test_matching_explicit_evidence_is_accepted(self) -> None: + self.evidence.write_text(json.dumps(self.body), encoding="utf-8") + result = self.run_verifier() + self.assertEqual(0, result.returncode, result.stderr) + + def test_missing_evidence_is_rejected(self) -> None: + result = self.run_verifier() + self.assertEqual(1, result.returncode) + self.assertIn("workload oracle evidence rejected", result.stderr) + + def test_tampered_model_identity_is_rejected(self) -> None: + self.body["model_sha256"] = "b" * 64 + self.evidence.write_text(json.dumps(self.body), encoding="utf-8") + result = self.run_verifier() + self.assertEqual(1, result.returncode) + self.assertIn("model_sha256 does not match", result.stderr) + + def test_smoke_only_output_cannot_certify_oracle_lane(self) -> None: + self.body["comparison"] = "embedding OpenAI HTTP smoke passed" + self.evidence.write_text(json.dumps(self.body), encoding="utf-8") + result = self.run_verifier() + self.assertEqual(1, result.returncode) + self.assertIn("lacks an explicit comparator pass", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_workload_monolithic_oracle.py b/scripts/tests/test_workload_monolithic_oracle.py new file mode 100644 index 0000000000..06af950cef --- /dev/null +++ b/scripts/tests/test_workload_monolithic_oracle.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path +import subprocess +import sys +import unittest +from unittest.mock import patch + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts" / "ci-workload-monolithic-oracle.py" +sys.path.insert(0, str(SCRIPT.parent)) +SPEC = importlib.util.spec_from_file_location("workload_monolithic_oracle", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +oracle = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(oracle) + + +class WorkloadMonolithicOracleTests(unittest.TestCase): + def test_embedding_requires_dimension_and_numeric_parity(self) -> None: + reference = { + "data": [ + {"index": index, "embedding": [1.0, 0.0]} + for index in range(len(oracle.EMBEDDING_INPUTS)) + ] + } + self.assertIn("max_abs_delta=0", oracle.compare_embeddings(reference, reference)) + changed = { + "data": [ + {"index": index, "embedding": [0.9, 0.1]} + for index in range(len(oracle.EMBEDDING_INPUTS)) + ] + } + with self.assertRaisesRegex(RuntimeError, "differs from monolithic reference"): + oracle.compare_embeddings(changed, reference) + changed["data"][0]["embedding"] = [1.0] + with self.assertRaisesRegex(RuntimeError, "dimensions differ"): + oracle.compare_embeddings(changed, reference) + + def test_embedding_oracle_checks_each_single_input_after_batch(self) -> None: + batch = {"data": [ + {"index": index, "embedding": [1.0, 0.0]} + for index in range(len(oracle.EMBEDDING_INPUTS)) + ]} + single = {"data": [{"index": 0, "embedding": [1.0, 0.0]}]} + divergent = {"data": [{"index": 0, "embedding": [0.0, 1.0]}]} + responses = [batch, batch, single, single, single, divergent, single, single] + with patch.object(oracle, "request_json", side_effect=responses) as request: + with self.assertRaisesRegex(RuntimeError, r"single\[1\]"): + oracle.run_embedding_oracle("http://candidate", "http://oracle", "fixture") + self.assertEqual(8, request.call_count) + self.assertEqual(oracle.EMBEDDING_INPUTS[0], request.call_args_list[2].args[2]["input"]) + + def test_rerank_requires_same_scores_and_order(self) -> None: + reference = {"results": [ + {"index": 0, "relevance_score": 2.0}, + {"index": 1, "relevance_score": -1.0}, + ]} + self.assertIn("max_abs_delta=0", oracle.compare_rerank(reference, reference)) + changed = {"results": [ + {"index": 0, "relevance_score": 0.1}, + {"index": 1, "relevance_score": 0.2}, + ]} + with self.assertRaisesRegex(RuntimeError, "differs from monolithic reference"): + oracle.compare_rerank(changed, reference) + + def test_encoder_decoder_compares_normalized_text(self) -> None: + reference = {"choices": [{"text": "Das Haus ist wunderbar."}]} + equivalent = {"choices": [{"text": " Das Haus ist wunderbar.\n"}]} + self.assertIn("identical normalized text", oracle.compare_encoder_decoder(equivalent, reference)) + changed = {"choices": [{"text": "Das Auto ist wunderbar."}]} + with self.assertRaisesRegex(RuntimeError, "differs from monolithic reference"): + oracle.compare_encoder_decoder(changed, reference) + + def test_direct_monolithic_completion_strips_only_terminal_runner_marker(self) -> None: + result = subprocess.CompletedProcess( + args=["llama-completion"], returncode=0, + stdout=" Das Haus ist schön. [end of text]\n", stderr="model loaded", + ) + with patch.object(oracle.subprocess, "run", return_value=result) as run: + response = oracle.monolithic_completion("/bin/llama-completion", "/model.gguf") + self.assertEqual("Das Haus ist schön.", response["choices"][0]["text"]) + self.assertIn("--no-display-prompt", run.call_args.args[0]) + self.assertIn("--temp", run.call_args.args[0]) + + def test_direct_monolithic_completion_rejects_empty_or_failed_output(self) -> None: + empty = subprocess.CompletedProcess(args=[], returncode=0, + stdout=" [end of text]\n", stderr="") + with patch.object(oracle.subprocess, "run", return_value=empty): + with self.assertRaisesRegex(RuntimeError, "produced no text"): + oracle.monolithic_completion("/bin/llama-completion", "/model.gguf") + failed = subprocess.CompletedProcess(args=[], returncode=1, stdout="", stderr="bad") + with patch.object(oracle.subprocess, "run", return_value=failed): + with self.assertRaisesRegex(RuntimeError, "exited 1"): + oracle.monolithic_completion("/bin/llama-completion", "/model.gguf") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/verify-workload-oracle-evidence.py b/scripts/verify-workload-oracle-evidence.py new file mode 100644 index 0000000000..39beb3b324 --- /dev/null +++ b/scripts/verify-workload-oracle-evidence.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Verify an observed non-chat oracle result before battery certification.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import sys + + +ORACLE_EXECUTABLE = { + "embedding": "llama-server", + "rerank": "llama-server", + "encoder_decoder": "llama-completion", + "ocr": "llama-server", + "speech_synthesis": "llama-tts", + "speech_recognition": "llama-server", +} + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def verify(args: argparse.Namespace) -> None: + evidence = json.loads(args.evidence.read_text(encoding="utf-8")) + if not isinstance(evidence, dict): + raise ValueError("oracle evidence must be an object") + expected = { + "status": "pass", + "class": args.model_class, + "smoke_lane": args.smoke_lane, + "oracle_lane": args.oracle_lane, + "model_id": args.model_id, + "model_sha256": sha256(args.model_path), + "projector_sha256": sha256(args.projector_path) if args.projector_path else None, + "oracle_executable": ORACLE_EXECUTABLE[args.model_class], + "oracle_executable_sha256": sha256(args.oracle_executable), + "candidate_executable_sha256": sha256(args.candidate_executable), + "pinned_patch_sha": args.pinned_patch_sha, + } + for key, value in expected.items(): + if evidence.get(key) != value: + raise ValueError(f"oracle evidence {key} does not match this run") + if args.oracle_executable.name != expected["oracle_executable"]: + raise ValueError("wrong oracle executable for workload class") + comparison = evidence.get("comparison") + if not isinstance(comparison, str) or not comparison.startswith( + f"{args.model_class} local-monolithic oracle passed: " + ): + raise ValueError("oracle evidence lacks an explicit comparator pass") + if args.model_class == "speech_synthesis" and not isinstance(evidence.get("metrics"), dict): + raise ValueError("TTS oracle evidence lacks PCM metrics") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--evidence", required=True, type=Path) + parser.add_argument("--class", dest="model_class", required=True, choices=ORACLE_EXECUTABLE) + parser.add_argument("--smoke-lane", required=True) + parser.add_argument("--oracle-lane", required=True) + parser.add_argument("--model-id", required=True) + parser.add_argument("--model-path", required=True, type=Path) + parser.add_argument("--projector-path", type=Path) + parser.add_argument("--candidate-executable", required=True, type=Path) + parser.add_argument("--oracle-executable", required=True, type=Path) + parser.add_argument("--pinned-patch-sha", required=True) + args = parser.parse_args() + try: + verify(args) + except (OSError, ValueError, KeyError, json.JSONDecodeError) as error: + print(f"workload oracle evidence rejected: {error}", file=sys.stderr) + return 1 + print(f"verified {args.model_class} local-monolithic oracle evidence") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/workload_fixtures.py b/scripts/workload_fixtures.py new file mode 100644 index 0000000000..500511261f --- /dev/null +++ b/scripts/workload_fixtures.py @@ -0,0 +1,15 @@ +"""Shared deterministic prompts for non-chat smoke and reference comparisons.""" + +EMBEDDING_INPUTS = ( + "search_query: distributed GPU inference", + "search_document: GPUs share one language model over a mesh", + "search_document: A recipe for tomato soup", +) + +RERANK_QUERY = "distributed GPU inference" +RERANK_DOCUMENTS = ( + "GPUs share one language model over a mesh", + "A recipe for tomato soup", +) + +ENCODER_DECODER_PROMPT = "translate English to German: The house is wonderful." diff --git a/scripts/write-workload-oracle-evidence.py b/scripts/write-workload-oracle-evidence.py new file mode 100644 index 0000000000..5d606983eb --- /dev/null +++ b/scripts/write-workload-oracle-evidence.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Persist an explicit, identity-bound workload oracle pass for the battery.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import sys + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def write_evidence(args: argparse.Namespace) -> None: + lines = [ + line.strip() + for line in args.comparison_log.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + if not lines or not lines[-1].startswith( + f"{args.model_class} local-monolithic oracle passed: " + ): + raise ValueError("oracle comparator did not emit an explicit class-specific pass") + evidence: dict[str, object] = { + "status": "pass", + "class": args.model_class, + "smoke_lane": args.smoke_lane, + "oracle_lane": args.smoke_lane.replace("-smoke", "-oracle"), + "model_id": args.model_id, + "model_sha256": args.model_sha256, + "projector_sha256": sha256(args.projector_path) if args.projector_path else None, + "oracle_executable": args.oracle_executable.name, + "oracle_executable_sha256": sha256(args.oracle_executable), + "candidate_executable_sha256": sha256(args.candidate_executable), + "pinned_patch_sha": args.pinned_patch_sha, + "comparison": lines[-1], + } + if args.model_class == "speech_synthesis": + result = json.loads((args.work_dir / "tts-oracle-result.json").read_text(encoding="utf-8")) + if result.get("status") != "pass" or result.get("pinned_patch_sha") != args.pinned_patch_sha: + raise ValueError("TTS comparator result is missing or does not match the pinned patch") + if not isinstance(result.get("metrics"), dict): + raise ValueError("TTS comparator result lacks PCM metrics") + evidence["metrics"] = result["metrics"] + args.output.write_text(json.dumps(evidence, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--comparison-log", required=True, type=Path) + parser.add_argument("--class", dest="model_class", required=True) + parser.add_argument("--smoke-lane", required=True) + parser.add_argument("--model-id", required=True) + parser.add_argument("--model-sha256", required=True) + parser.add_argument("--projector-path", type=Path) + parser.add_argument("--candidate-executable", required=True, type=Path) + parser.add_argument("--oracle-executable", required=True, type=Path) + parser.add_argument("--pinned-patch-sha", required=True) + parser.add_argument("--work-dir", required=True, type=Path) + args = parser.parse_args() + try: + write_evidence(args) + except (OSError, ValueError) as error: + print(f"workload oracle evidence not written: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/third_party/llama.cpp/patches/0024-skippy-define-non-chat-workload-ABI-and-execution.patch b/third_party/llama.cpp/patches/0024-skippy-define-non-chat-workload-ABI-and-execution.patch new file mode 100644 index 0000000000..386e150a05 --- /dev/null +++ b/third_party/llama.cpp/patches/0024-skippy-define-non-chat-workload-ABI-and-execution.patch @@ -0,0 +1,630 @@ +From 4bee60c00508ca2f6037fb8d4a042c961f26a697 Mon Sep 17 00:00:00 2001 +From: Mesh-LLM CI +Date: Sat, 12 Sep 2026 01:35:50 -0700 +Subject: [PATCH 24/25] skippy: define non-chat workload ABI and execution + +--- + include/skippy.h | 1 + + include/skippy/common.h | 3 +- + include/skippy/workloads.h | 87 +++++++ + src/CMakeLists.txt | 1 + + src/skippy/abi.cpp | 3 +- + src/skippy/rerank_template.h | 36 +++ + src/skippy/workloads.cpp | 356 ++++++++++++++++++++++++++ + tests/CMakeLists.txt | 1 + + tests/test-skippy-rerank-template.cpp | 34 +++ + 9 files changed, 520 insertions(+), 2 deletions(-) + create mode 100644 include/skippy/workloads.h + create mode 100644 src/skippy/rerank_template.h + create mode 100644 src/skippy/workloads.cpp + create mode 100644 tests/test-skippy-rerank-template.cpp + +diff --git a/include/skippy.h b/include/skippy.h +index 33a47b244..dc8cbb431 100644 +--- a/include/skippy.h ++++ b/include/skippy.h +@@ -24,5 +24,6 @@ + #include "skippy/tokenization.h" + #include "skippy/model_package.h" + #include "skippy/signals.h" ++#include "skippy/workloads.h" + + #endif // SKIPPY_H +diff --git a/include/skippy/common.h b/include/skippy/common.h +index 65cfa2ce8..e921a5ba4 100644 +--- a/include/skippy/common.h ++++ b/include/skippy/common.h +@@ -46,7 +46,7 @@ extern "C" { + + #define SKIPPY_ABI_VERSION_MAJOR 0 + #define SKIPPY_ABI_VERSION_MINOR 1 +-#define SKIPPY_ABI_VERSION_PATCH 53 ++#define SKIPPY_ABI_VERSION_PATCH 54 + + #if defined(_MSC_VER) + #define SKIPPY_DEPRECATED(message) __declspec(deprecated(message)) +@@ -88,6 +88,7 @@ enum skippy_feature { + SKIPPY_FEATURE_ITERATION_BATCH = 1 << 28, + SKIPPY_FEATURE_ACTIVATION_BOUNDARY = 1 << 29, + SKIPPY_FEATURE_MODEL_SOURCE = 1 << 30, ++ SKIPPY_FEATURE_NON_CHAT_WORKLOADS = UINT64_C(1) << 31, + }; + + enum skippy_status { +diff --git a/include/skippy/workloads.h b/include/skippy/workloads.h +new file mode 100644 +index 000000000..0ee115741 +--- /dev/null ++++ b/include/skippy/workloads.h +@@ -0,0 +1,87 @@ ++#ifndef SKIPPY_WORKLOADS_H ++#define SKIPPY_WORKLOADS_H ++ ++/** ++ * @file skippy/workloads.h ++ * @brief Full-model embedding, reranking, and encoder-decoder execution. ++ * ++ * These operations deliberately reject filtered stage models. Their explicit ++ * workload descriptor lets callers fail closed instead of inferring support ++ * from a model name or architecture family. ++ */ ++ ++#include "common.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++struct skippy_model; ++struct skippy_session; ++ ++#define SKIPPY_WORKLOAD_INFO_V1_ABI_VERSION 1 ++ ++enum skippy_workload_kind { ++ SKIPPY_WORKLOAD_CAUSAL_GENERATION = 0, ++ SKIPPY_WORKLOAD_EMBEDDING = 1, ++ SKIPPY_WORKLOAD_RERANK = 2, ++ SKIPPY_WORKLOAD_ENCODER_DECODER = 3, ++}; ++ ++enum skippy_workload_pooling { ++ SKIPPY_WORKLOAD_POOLING_UNSPECIFIED = -1, ++ SKIPPY_WORKLOAD_POOLING_NONE = 0, ++ SKIPPY_WORKLOAD_POOLING_MEAN = 1, ++ SKIPPY_WORKLOAD_POOLING_CLS = 2, ++ SKIPPY_WORKLOAD_POOLING_LAST = 3, ++ SKIPPY_WORKLOAD_POOLING_RANK = 4, ++}; ++ ++/** @brief Runtime-probed workload properties for an opened model. */ ++struct skippy_workload_info_v1 { ++ uint32_t abi_version; ++ uint32_t struct_size; ++ enum skippy_workload_kind kind; ++ enum skippy_workload_pooling pooling; ++ uint32_t output_dimensions; ++ uint32_t classifier_outputs; ++ bool has_encoder; ++ bool has_decoder; ++ bool full_model_only; ++ uint8_t reserved0; ++}; ++ ++/** @brief Describes the workload implemented by an opened model. */ ++LLAMA_API enum skippy_status skippy_model_workload_info_v1(const struct skippy_model * model, ++ struct skippy_workload_info_v1 * out_info, ++ struct skippy_error ** out_error); ++ ++/** @brief Computes one normalized embedding for a tokenized input. */ ++LLAMA_API enum skippy_status skippy_session_embed(struct skippy_session * session, ++ const llama_token * token_ids, ++ size_t token_count, ++ float * output, ++ size_t output_capacity, ++ size_t * out_dimensions, ++ struct skippy_error ** out_error); ++ ++/** @brief Computes one scalar relevance score for a query-document pair. */ ++LLAMA_API enum skippy_status skippy_session_rerank(struct skippy_session * session, ++ const char * query, ++ const char * document, ++ float * out_score, ++ size_t * out_token_count, ++ struct skippy_error ** out_error); ++ ++/** @brief Encodes an encoder-decoder prompt and returns its first decoder token. */ ++LLAMA_API enum skippy_status skippy_session_encode_prompt(struct skippy_session * session, ++ const llama_token * token_ids, ++ size_t token_count, ++ llama_token * out_decoder_start_token, ++ struct skippy_error ** out_error); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif // SKIPPY_WORKLOADS_H +diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt +index 118134918..bfc60ceb5 100644 +--- a/src/CMakeLists.txt ++++ b/src/CMakeLists.txt +@@ -49,6 +49,7 @@ add_library(llama + skippy/model_source.cpp + skippy/session.cpp + skippy/verification.cpp ++ skippy/workloads.cpp + skippy/abi.cpp + skippy/block_boundaries.cpp + skippy/devices.cpp +diff --git a/src/skippy/abi.cpp b/src/skippy/abi.cpp +index ec2b47444..5fff65721 100644 +--- a/src/skippy/abi.cpp ++++ b/src/skippy/abi.cpp +@@ -39,7 +39,8 @@ uint64_t skippy_abi_features(void) { + SKIPPY_FEATURE_INKLING_MTP_MM | + SKIPPY_FEATURE_ITERATION_BATCH | + SKIPPY_FEATURE_ACTIVATION_BOUNDARY | +- SKIPPY_FEATURE_MODEL_SOURCE; ++ SKIPPY_FEATURE_MODEL_SOURCE | ++ SKIPPY_FEATURE_NON_CHAT_WORKLOADS; + } + + void skippy_error_free(struct skippy_error * error) { +diff --git a/src/skippy/rerank_template.h b/src/skippy/rerank_template.h +new file mode 100644 +index 000000000..f0c3b2ea7 +--- /dev/null ++++ b/src/skippy/rerank_template.h +@@ -0,0 +1,36 @@ ++#pragma once ++ ++#include ++#include ++#include ++#include ++ ++// Substitute only markers from the model-authored template. User-supplied ++// query/document bytes are appended verbatim and must never be parsed again. ++inline std::string skippy_render_rerank_template( ++ std::string_view prompt_template, ++ std::string_view query, ++ std::string_view document) { ++ constexpr std::string_view query_marker = "{query}"; ++ constexpr std::string_view document_marker = "{document}"; ++ std::string rendered; ++ std::size_t cursor = 0; ++ while (cursor < prompt_template.size()) { ++ const std::size_t query_pos = prompt_template.find(query_marker, cursor); ++ const std::size_t document_pos = prompt_template.find(document_marker, cursor); ++ const std::size_t next = std::min(query_pos, document_pos); ++ if (next == std::string_view::npos) { ++ rendered.append(prompt_template.data() + cursor, prompt_template.size() - cursor); ++ break; ++ } ++ rendered.append(prompt_template.data() + cursor, next - cursor); ++ if (next == query_pos) { ++ rendered.append(query.data(), query.size()); ++ cursor = next + query_marker.size(); ++ } else { ++ rendered.append(document.data(), document.size()); ++ cursor = next + document_marker.size(); ++ } ++ } ++ return rendered; ++} +diff --git a/src/skippy/workloads.cpp b/src/skippy/workloads.cpp +new file mode 100644 +index 000000000..81ceb3473 +--- /dev/null ++++ b/src/skippy/workloads.cpp +@@ -0,0 +1,356 @@ ++#include "skippy/workloads.h" ++ ++#include "llama.h" ++#include "llama-model.h" ++#include "skippy/errors.h" ++#include "skippy/rerank_template.h" ++#include "skippy/runtime_support.h" ++#include "skippy/runtime_types.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++namespace { ++ ++enum skippy_workload_kind workload_kind(const skippy_model * model) { ++ const bool has_encoder = llama_model_has_encoder(model->model); ++ const bool has_decoder = llama_model_has_decoder(model->model); ++ const enum llama_pooling_type pooling = llama_pooling_type(model->ctx); ++ ++ if (model->model->cls != nullptr || model->model->cls_out != nullptr || pooling == LLAMA_POOLING_TYPE_RANK) { ++ return SKIPPY_WORKLOAD_RERANK; ++ } ++ if (has_encoder && has_decoder) { ++ return SKIPPY_WORKLOAD_ENCODER_DECODER; ++ } ++ if (has_encoder || (pooling != LLAMA_POOLING_TYPE_NONE && pooling != LLAMA_POOLING_TYPE_UNSPECIFIED)) { ++ return SKIPPY_WORKLOAD_EMBEDDING; ++ } ++ return SKIPPY_WORKLOAD_CAUSAL_GENERATION; ++} ++ ++bool is_full_model(const skippy_model * model) { ++ return model != nullptr && !model->config.filter_tensors_on_load; ++} ++ ++enum skippy_status require_workload(skippy_session * session, ++ enum skippy_workload_kind required, ++ skippy_error ** out_error) { ++ if (session == nullptr || session->stage_model == nullptr || session->stage_model->model == nullptr || ++ session->ctx == nullptr) { ++ skippy_set_error(out_error, SKIPPY_STATUS_INVALID_ARGUMENT, "an active session is required"); ++ return SKIPPY_STATUS_INVALID_ARGUMENT; ++ } ++ if (!is_full_model(session->stage_model)) { ++ skippy_set_error(out_error, SKIPPY_STATUS_UNSUPPORTED, "non-chat workloads currently require a full model"); ++ return SKIPPY_STATUS_UNSUPPORTED; ++ } ++ if (workload_kind(session->stage_model) != required) { ++ skippy_set_error(out_error, SKIPPY_STATUS_UNSUPPORTED, "model does not implement the requested workload"); ++ return SKIPPY_STATUS_UNSUPPORTED; ++ } ++ return SKIPPY_STATUS_OK; ++} ++ ++enum skippy_status validate_tokens(skippy_session * session, ++ const llama_token * token_ids, ++ size_t token_count, ++ skippy_error ** out_error) { ++ if (token_ids == nullptr || token_count == 0) { ++ skippy_set_error(out_error, SKIPPY_STATUS_INVALID_ARGUMENT, "at least one token is required"); ++ return SKIPPY_STATUS_INVALID_ARGUMENT; ++ } ++ if (token_count > static_cast(std::numeric_limits::max()) || ++ token_count > static_cast(llama_n_batch(session->ctx))) { ++ skippy_set_error(out_error, SKIPPY_STATUS_INVALID_ARGUMENT, "token count exceeds the model batch capacity"); ++ return SKIPPY_STATUS_INVALID_ARGUMENT; ++ } ++ return SKIPPY_STATUS_OK; ++} ++ ++void clear_workload_state(skippy_session * session) { ++ skippy_synchronize_context_if_pending(session->stage_model); ++ if (llama_memory_t memory = llama_get_memory(session->ctx)) { ++ llama_memory_clear(memory, true); ++ } ++ session->n_past = 0; ++ session->token_history.clear(); ++ session->signal_history.clear(); ++} ++ ++llama_batch make_embedding_batch(skippy_session * session, const llama_token * token_ids, size_t token_count) { ++ llama_batch batch = llama_batch_init(static_cast(token_count), 0, 1); ++ batch.n_tokens = static_cast(token_count); ++ for (int32_t index = 0; index < batch.n_tokens; ++index) { ++ batch.token[index] = token_ids[index]; ++ batch.pos[index] = index; ++ batch.n_seq_id[index] = 1; ++ batch.seq_id[index][0] = session->seq_id; ++ batch.logits[index] = 1; ++ } ++ return batch; ++} ++ ++enum skippy_status decode_embedding(skippy_session * session, ++ const llama_token * token_ids, ++ size_t token_count, ++ const float ** out_embedding, ++ skippy_error ** out_error) { ++ clear_workload_state(session); ++ llama_batch batch = make_embedding_batch(session, token_ids, token_count); ++ const int32_t decode_status = llama_decode(session->ctx, batch); ++ llama_batch_free(batch); ++ if (decode_status != 0) { ++ skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, "llama_decode failed for non-chat workload"); ++ return SKIPPY_STATUS_RUNTIME_ERROR; ++ } ++ skippy_mark_context_work_pending(session->stage_model); ++ ++ const enum llama_pooling_type pooling = llama_pooling_type(session->ctx); ++ const float * embedding = pooling == LLAMA_POOLING_TYPE_NONE ? ++ llama_get_embeddings_ith(session->ctx, static_cast(token_count - 1)) : ++ llama_get_embeddings_seq(session->ctx, session->seq_id); ++ skippy_mark_context_synchronized(session->stage_model); ++ if (embedding == nullptr) { ++ skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, "model produced no workload output"); ++ return SKIPPY_STATUS_RUNTIME_ERROR; ++ } ++ *out_embedding = embedding; ++ session->n_past = static_cast(token_count); ++ session->token_history.assign(token_ids, token_ids + token_count); ++ return SKIPPY_STATUS_OK; ++} ++ ++void normalize_embedding(const float * source, float * output, size_t dimensions) { ++ double squared_norm = 0.0; ++ for (size_t index = 0; index < dimensions; ++index) { ++ squared_norm += static_cast(source[index]) * source[index]; ++ } ++ const float scale = squared_norm > 0.0 ? static_cast(1.0 / std::sqrt(squared_norm)) : 1.0f; ++ for (size_t index = 0; index < dimensions; ++index) { ++ output[index] = source[index] * scale; ++ } ++} ++ ++std::string token_piece(const llama_vocab * vocab, llama_token token) { ++ char stack[128]; ++ int32_t length = llama_token_to_piece(vocab, token, stack, sizeof(stack), 0, true); ++ if (length >= 0) { ++ return std::string(stack, static_cast(length)); ++ } ++ std::string dynamic(static_cast(-length), '\0'); ++ length = llama_token_to_piece(vocab, token, dynamic.data(), static_cast(dynamic.size()), 0, true); ++ return length > 0 ? dynamic.substr(0, static_cast(length)) : std::string(); ++} ++ ++bool tokenize(const llama_vocab * vocab, const std::string & text, std::vector & tokens) { ++ if (text.size() > static_cast(std::numeric_limits::max())) { ++ return false; ++ } ++ const int32_t text_length = static_cast(text.size()); ++ int32_t count = llama_tokenize(vocab, text.data(), text_length, nullptr, 0, true, true); ++ if (count >= 0) { ++ tokens.resize(static_cast(count)); ++ } else { ++ tokens.resize(static_cast(-count)); ++ } ++ count = ++ llama_tokenize(vocab, text.data(), text_length, tokens.data(), static_cast(tokens.size()), true, true); ++ if (count < 0) { ++ return false; ++ } ++ tokens.resize(static_cast(count)); ++ return !tokens.empty(); ++} ++ ++std::string rerank_prompt(const llama_model * model, const char * query, const char * document) { ++ if (const char * prompt_template = llama_model_chat_template(model, "rerank")) { ++ return skippy_render_rerank_template(prompt_template, query, document); ++ } ++ ++ const llama_vocab * vocab = llama_model_get_vocab(model); ++ std::string prompt(query); ++ if (llama_vocab_get_add_eos(vocab)) { ++ prompt += token_piece(vocab, llama_vocab_eos(vocab)); ++ } ++ if (llama_vocab_get_add_sep(vocab)) { ++ prompt += token_piece(vocab, llama_vocab_sep(vocab)); ++ } ++ prompt += document; ++ return prompt; ++} ++ ++} // namespace ++ ++extern "C" { ++ ++enum skippy_status skippy_model_workload_info_v1(const skippy_model * model, ++ skippy_workload_info_v1 * out_info, ++ skippy_error ** out_error) { ++ if (model == nullptr || model->model == nullptr || model->ctx == nullptr || out_info == nullptr) { ++ skippy_set_error(out_error, SKIPPY_STATUS_INVALID_ARGUMENT, "model and out_info are required"); ++ return SKIPPY_STATUS_INVALID_ARGUMENT; ++ } ++ if (out_info->abi_version != SKIPPY_WORKLOAD_INFO_V1_ABI_VERSION || ++ out_info->struct_size != sizeof(skippy_workload_info_v1)) { ++ skippy_set_error(out_error, SKIPPY_STATUS_INVALID_ARGUMENT, "incompatible workload descriptor ABI"); ++ return SKIPPY_STATUS_INVALID_ARGUMENT; ++ } ++ ++ const skippy_workload_kind kind = workload_kind(model); ++ const uint32_t output_dimensions = kind == SKIPPY_WORKLOAD_EMBEDDING ? ++ static_cast(std::max(llama_model_n_embd_out(model->model), 0)) : ++ 0; ++ const uint32_t classifier_outputs = kind == SKIPPY_WORKLOAD_RERANK ? llama_model_n_cls_out(model->model) : 0; ++ *out_info = { ++ SKIPPY_WORKLOAD_INFO_V1_ABI_VERSION, ++ static_cast(sizeof(skippy_workload_info_v1)), ++ kind, ++ static_cast(llama_pooling_type(model->ctx)), ++ output_dimensions, ++ classifier_outputs, ++ llama_model_has_encoder(model->model), ++ llama_model_has_decoder(model->model), ++ true, ++ 0, ++ }; ++ return skippy_success(out_error); ++} ++ ++enum skippy_status skippy_session_embed(skippy_session * session, ++ const llama_token * token_ids, ++ size_t token_count, ++ float * output, ++ size_t output_capacity, ++ size_t * out_dimensions, ++ skippy_error ** out_error) { ++ if (out_dimensions != nullptr) { ++ *out_dimensions = 0; ++ } ++ enum skippy_status status = require_workload(session, SKIPPY_WORKLOAD_EMBEDDING, out_error); ++ if (status != SKIPPY_STATUS_OK) { ++ return status; ++ } ++ if (output == nullptr || out_dimensions == nullptr) { ++ skippy_set_error(out_error, SKIPPY_STATUS_INVALID_ARGUMENT, "output and out_dimensions are required"); ++ return SKIPPY_STATUS_INVALID_ARGUMENT; ++ } ++ status = validate_tokens(session, token_ids, token_count, out_error); ++ if (status != SKIPPY_STATUS_OK) { ++ return status; ++ } ++ const int32_t native_dimensions = llama_model_n_embd_out(session->stage_model->model); ++ if (native_dimensions <= 0) { ++ skippy_set_error(out_error, SKIPPY_STATUS_MODEL_ERROR, "model has no embedding output dimensions"); ++ return SKIPPY_STATUS_MODEL_ERROR; ++ } ++ const size_t dimensions = static_cast(native_dimensions); ++ *out_dimensions = dimensions; ++ if (output_capacity < dimensions) { ++ skippy_set_error(out_error, SKIPPY_STATUS_BUFFER_TOO_SMALL, "embedding output buffer is too small"); ++ return SKIPPY_STATUS_BUFFER_TOO_SMALL; ++ } ++ ++ const float * embedding = nullptr; ++ status = decode_embedding(session, token_ids, token_count, &embedding, out_error); ++ if (status != SKIPPY_STATUS_OK) { ++ return status; ++ } ++ normalize_embedding(embedding, output, dimensions); ++ return skippy_success(out_error); ++} ++ ++enum skippy_status skippy_session_rerank(skippy_session * session, ++ const char * query, ++ const char * document, ++ float * out_score, ++ size_t * out_token_count, ++ skippy_error ** out_error) { ++ if (out_token_count != nullptr) { ++ *out_token_count = 0; ++ } ++ enum skippy_status status = require_workload(session, SKIPPY_WORKLOAD_RERANK, out_error); ++ if (status != SKIPPY_STATUS_OK) { ++ return status; ++ } ++ if (query == nullptr || document == nullptr || out_score == nullptr || out_token_count == nullptr) { ++ skippy_set_error(out_error, SKIPPY_STATUS_INVALID_ARGUMENT, ++ "query, document, and output pointers are required"); ++ return SKIPPY_STATUS_INVALID_ARGUMENT; ++ } ++ if (llama_model_n_cls_out(session->stage_model->model) != 1) { ++ skippy_set_error(out_error, SKIPPY_STATUS_UNSUPPORTED, "rerank API requires a single classifier output"); ++ return SKIPPY_STATUS_UNSUPPORTED; ++ } ++ ++ std::vector tokens; ++ const std::string prompt = rerank_prompt(session->stage_model->model, query, document); ++ if (!tokenize(llama_model_get_vocab(session->stage_model->model), prompt, tokens)) { ++ skippy_set_error(out_error, SKIPPY_STATUS_MODEL_ERROR, "failed to tokenize rerank input"); ++ return SKIPPY_STATUS_MODEL_ERROR; ++ } ++ status = validate_tokens(session, tokens.data(), tokens.size(), out_error); ++ if (status != SKIPPY_STATUS_OK) { ++ return status; ++ } ++ ++ const float * ranks = nullptr; ++ status = decode_embedding(session, tokens.data(), tokens.size(), &ranks, out_error); ++ if (status != SKIPPY_STATUS_OK) { ++ return status; ++ } ++ *out_score = ranks[0]; ++ *out_token_count = tokens.size(); ++ return skippy_success(out_error); ++} ++ ++enum skippy_status skippy_session_encode_prompt(skippy_session * session, ++ const llama_token * token_ids, ++ size_t token_count, ++ llama_token * out_decoder_start_token, ++ skippy_error ** out_error) { ++ enum skippy_status status = require_workload(session, SKIPPY_WORKLOAD_ENCODER_DECODER, out_error); ++ if (status != SKIPPY_STATUS_OK) { ++ return status; ++ } ++ if (out_decoder_start_token == nullptr) { ++ skippy_set_error(out_error, SKIPPY_STATUS_INVALID_ARGUMENT, "out_decoder_start_token is required"); ++ return SKIPPY_STATUS_INVALID_ARGUMENT; ++ } ++ status = validate_tokens(session, token_ids, token_count, out_error); ++ if (status != SKIPPY_STATUS_OK) { ++ return status; ++ } ++ if (token_count > static_cast(llama_n_ubatch(session->ctx))) { ++ skippy_set_error(out_error, SKIPPY_STATUS_INVALID_ARGUMENT, ++ "encoder input exceeds the physical batch capacity"); ++ return SKIPPY_STATUS_INVALID_ARGUMENT; ++ } ++ ++ clear_workload_state(session); ++ llama_batch batch = llama_batch_get_one(const_cast(token_ids), static_cast(token_count)); ++ if (llama_encode(session->ctx, batch) != 0) { ++ skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, "llama_encode failed for encoder-decoder workload"); ++ return SKIPPY_STATUS_RUNTIME_ERROR; ++ } ++ skippy_mark_context_work_pending(session->stage_model); ++ ++ llama_token start = llama_model_decoder_start_token(session->stage_model->model); ++ if (start == LLAMA_TOKEN_NULL) { ++ start = llama_vocab_bos(llama_model_get_vocab(session->stage_model->model)); ++ } ++ if (start == LLAMA_TOKEN_NULL) { ++ skippy_set_error(out_error, SKIPPY_STATUS_MODEL_ERROR, "model has no decoder start or BOS token"); ++ return SKIPPY_STATUS_MODEL_ERROR; ++ } ++ *out_decoder_start_token = start; ++ return skippy_success(out_error); ++} ++ ++} // extern "C" +diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt +index 5a45037bb..dc52da145 100644 +--- a/tests/CMakeLists.txt ++++ b/tests/CMakeLists.txt +@@ -150,6 +150,7 @@ endif () + + llama_build(test-recurrent-state-rollback.cpp) + llama_build(test-save-load-state.cpp) ++llama_build_and_test(test-skippy-rerank-template.cpp) + + if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) + llama_build_and_test(test-skippy-kv-page-export.cpp) +diff --git a/tests/test-skippy-rerank-template.cpp b/tests/test-skippy-rerank-template.cpp +new file mode 100644 +index 000000000..d6f9570ba +--- /dev/null ++++ b/tests/test-skippy-rerank-template.cpp +@@ -0,0 +1,34 @@ ++#include "../src/skippy/rerank_template.h" ++ ++#include ++#include ++ ++static bool expect_rendered( ++ std::string_view prompt_template, ++ std::string_view query, ++ std::string_view document, ++ std::string_view expected) { ++ const auto rendered = skippy_render_rerank_template(prompt_template, query, document); ++ if (std::string_view(rendered) == expected) { ++ return true; ++ } ++ std::fprintf(stderr, "rerank template rendered unexpected text: %s\n", rendered.c_str()); ++ return false; ++} ++ ++int main() { ++ if (!expect_rendered( ++ "Q={query}|D={document}|Q={query}", ++ "literal {document}", ++ "literal {query}", ++ "Q=literal {document}|D=literal {query}|Q=literal {document}")) { ++ return 1; ++ } ++ if (!expect_rendered("{query}{document}", "a", "b", "ab")) { ++ return 2; ++ } ++ if (!expect_rendered("no markers", "a", "b", "no markers")) { ++ return 3; ++ } ++ return 0; ++} +-- +2.54.0 (Apple Git-157) diff --git a/third_party/llama.cpp/patches/0025-skippy-configure-non-chat-model-loading.patch b/third_party/llama.cpp/patches/0025-skippy-configure-non-chat-model-loading.patch new file mode 100644 index 0000000000..4c76acfe00 --- /dev/null +++ b/third_party/llama.cpp/patches/0025-skippy-configure-non-chat-model-loading.patch @@ -0,0 +1,55 @@ +From 6a85b4d44ea932a6ed644e5931b05a746066cec6 Mon Sep 17 00:00:00 2001 +From: Mesh-LLM CI +Date: Sat, 12 Sep 2026 01:36:02 -0700 +Subject: [PATCH 25/25] skippy: configure non-chat model loading + +--- + src/skippy/model_loading.cpp | 23 +++++++++++++++++++++-- + 1 file changed, 21 insertions(+), 2 deletions(-) + +diff --git a/src/skippy/model_loading.cpp b/src/skippy/model_loading.cpp +index 13dc3d8ee..ed859029d 100644 +--- a/src/skippy/model_loading.cpp ++++ b/src/skippy/model_loading.cpp +@@ -564,9 +564,19 @@ enum skippy_status skippy_finish_model_open( + } + } + +- const uint32_t lane_count = config != nullptr && config->lane_count > 0 ++ const bool encoder_decoder = llama_model_has_encoder(model) && llama_model_has_decoder(model); ++ const bool classifier_workload = model->cls != nullptr || model->cls_out != nullptr; ++ const bool embedding_workload = !encoder_decoder && ++ (classifier_workload || ++ (llama_model_has_encoder(model) || ++ (model->hparams.pooling_type != LLAMA_POOLING_TYPE_NONE && ++ model->hparams.pooling_type != LLAMA_POOLING_TYPE_UNSPECIFIED))); ++ const uint32_t configured_lane_count = config != nullptr && config->lane_count > 0 + ? static_cast(config->lane_count) + : 1; ++ // Encoder output is context-global in llama.cpp. Until that state can be ++ // isolated per sequence, serialize encoder-decoder work on one lane. ++ const uint32_t lane_count = encoder_decoder ? 1 : configured_lane_count; + uint32_t context_size_per_lane = 0; + uint32_t context_size_total = 0; + if (!skippy_context_capacity(config, lane_count, context_size_per_lane, context_size_total)) { +@@ -606,7 +616,16 @@ enum skippy_status skippy_finish_model_open( + params.type_k = config != nullptr && config->cache_type_k > 0 ? static_cast(config->cache_type_k) : GGML_TYPE_F16; + params.type_v = config != nullptr && config->cache_type_v > 0 ? static_cast(config->cache_type_v) : GGML_TYPE_F16; + params.flash_attn_type = config != nullptr ? static_cast(config->flash_attn_type) : LLAMA_FLASH_ATTN_TYPE_AUTO; +- params.embeddings = config != nullptr && config->filter_tensors_on_load && !config->include_output; ++ if (classifier_workload) { ++ params.pooling_type = LLAMA_POOLING_TYPE_RANK; ++ } ++ params.embeddings = embedding_workload || ++ (config != nullptr && config->filter_tensors_on_load && !config->include_output); ++ if (embedding_workload) { ++ // Non-causal embedding graphs must fit their logical batch in one ++ // physical batch, matching llama-embedding's runtime contract. ++ params.n_ubatch = params.n_batch; ++ } + const bool glm_dsa_op_timing_enabled = skippy_glm_dsa_op_timing_enabled(); + const bool glm_dsa_tensor_trace_enabled = skippy_glm_dsa_tensor_trace_enabled(); + if (model->arch == LLM_ARCH_GLM_DSA && (glm_dsa_op_timing_enabled || glm_dsa_tensor_trace_enabled)) { +-- +2.54.0 (Apple Git-157) + diff --git a/third_party/llama.cpp/patches/0026-skippy-honor-vocab-suppress-tokens-in-sampling.patch b/third_party/llama.cpp/patches/0026-skippy-honor-vocab-suppress-tokens-in-sampling.patch new file mode 100644 index 0000000000..c23a39c3d9 --- /dev/null +++ b/third_party/llama.cpp/patches/0026-skippy-honor-vocab-suppress-tokens-in-sampling.patch @@ -0,0 +1,348 @@ +From 0c66f101778b066262384f8adb30a066fae0e5e2 Mon Sep 17 00:00:00 2001 +From: Mesh-LLM CI +Date: Sat, 12 Sep 2026 04:07:46 -0700 +Subject: [PATCH] skippy: honor vocab suppress tokens in sampling + +--- + src/CMakeLists.txt | 1 + + src/skippy/sampling.cpp | 78 ++++++++++++------------- + src/skippy/sampling_internal.h | 13 +++++ + src/skippy/sampling_vocab.cpp | 63 ++++++++++++++++++++ + tests/CMakeLists.txt | 1 + + tests/test-skippy-sampling-suppress.cpp | 69 ++++++++++++++++++++++ + 6 files changed, 186 insertions(+), 39 deletions(-) + create mode 100644 src/skippy/sampling_vocab.cpp + create mode 100644 tests/test-skippy-sampling-suppress.cpp + +diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt +index bfc60ceb5..2aac07d2a 100644 +--- a/src/CMakeLists.txt ++++ b/src/CMakeLists.txt +@@ -60,6 +60,7 @@ add_library(llama + skippy/model_package.cpp + skippy/package_tensor_binding.cpp + skippy/sampling.cpp ++ skippy/sampling_vocab.cpp + skippy/stage_slice_plan.cpp + skippy/stage_plan.cpp + skippy/state.cpp +diff --git a/src/skippy/sampling.cpp b/src/skippy/sampling.cpp +index 8c87bbe07..a900f28ce 100644 +--- a/src/skippy/sampling.cpp ++++ b/src/skippy/sampling.cpp +@@ -170,6 +170,8 @@ static llama_token skippy_greedy_sample_context_with_eog_policy( + } + const llama_vocab * vocab = llama_model_get_vocab(model); + const int32_t n_vocab = llama_vocab_n_tokens(vocab); ++ int32_t suppress_token_count = 0; ++ const llama_token * suppress_tokens = llama_vocab_get_suppress_tokens(vocab, &suppress_token_count); + llama_synchronize(ctx); + const float * logits = llama_get_logits_ith(ctx, index); + if (logits == nullptr) { +@@ -187,24 +189,17 @@ static llama_token skippy_greedy_sample_context_with_eog_policy( + &accelerated_best_logit, + &best_index, + static_cast(n_vocab)); +- if (!ignore_eog || !llama_vocab_is_eog(vocab, static_cast(best_index))) { ++ const bool accelerated_suppressed = suppress_tokens != nullptr && suppress_token_count > 0 && ++ std::find(suppress_tokens, suppress_tokens + suppress_token_count, ++ static_cast(best_index)) != suppress_tokens + suppress_token_count; ++ if (!accelerated_suppressed && ++ (!ignore_eog || !llama_vocab_is_eog(vocab, static_cast(best_index)))) { + return static_cast(best_index); + } + #endif + +- llama_token best = 0; +- float best_logit = -std::numeric_limits::infinity(); +- for (int32_t token = 0; token < n_vocab; ++token) { +- if (ignore_eog && llama_vocab_is_eog(vocab, token)) { +- continue; +- } +- if (logits[token] > best_logit) { +- best_logit = logits[token]; +- best = token; +- } +- } +- +- return best; ++ return skippy_greedy_sample_allowed_logits( ++ logits, vocab, n_vocab, suppress_tokens, suppress_token_count, ignore_eog); + } + + llama_token skippy_greedy_sample_context( +@@ -480,29 +475,35 @@ bool skippy_reset_reusable_sampling(skippy_session * session) { + static std::vector skippy_sampling_logit_biases( + skippy_session * session, + const skippy_sampling_config * sampling) { +- const uint32_t count = std::min(sampling->logit_bias_count, SKIPPY_MAX_LOGIT_BIAS); +- std::vector biases(sampling->logit_bias, sampling->logit_bias + count); +- const bool ignore_eos = sampling->ignore_eos != 0 || +- (sampling->flags & SKIPPY_SAMPLING_FLAG_IGNORE_EOS) != 0; +- if (!ignore_eos) { +- return biases; +- } ++ std::vector biases; ++ if (sampling != nullptr) { ++ const uint32_t count = std::min(sampling->logit_bias_count, SKIPPY_MAX_LOGIT_BIAS); ++ biases.assign(sampling->logit_bias, sampling->logit_bias + count); ++ } ++ const bool ignore_eos = sampling != nullptr && ++ (sampling->ignore_eos != 0 || ++ (sampling->flags & SKIPPY_SAMPLING_FLAG_IGNORE_EOS) != 0); + const llama_vocab * vocab = llama_model_get_vocab(session->stage_model->model); +- const int32_t token_count = llama_vocab_n_tokens(vocab); +- for (llama_token token = 0; token < token_count; ++token) { +- if (!llama_vocab_is_eog(vocab, token)) { +- continue; +- } +- const auto found = std::find_if(biases.begin(), biases.end(), [token](const llama_logit_bias & bias) { +- return bias.token == token; +- }); +- if (found == biases.end()) { +- biases.push_back({token, -INFINITY}); +- } else { +- found->bias = -INFINITY; ++ if (ignore_eos) { ++ const int32_t token_count = llama_vocab_n_tokens(vocab); ++ for (llama_token token = 0; token < token_count; ++token) { ++ if (!llama_vocab_is_eog(vocab, token)) { ++ continue; ++ } ++ const auto found = std::find_if(biases.begin(), biases.end(), [token](const llama_logit_bias & bias) { ++ return bias.token == token; ++ }); ++ if (found == biases.end()) { ++ biases.push_back({token, -INFINITY}); ++ } else { ++ found->bias = -INFINITY; ++ } + } + } +- return biases; ++ int32_t suppress_token_count = 0; ++ const llama_token * suppress_tokens = llama_vocab_get_suppress_tokens(vocab, &suppress_token_count); ++ return skippy_add_vocab_suppress_token_biases( ++ std::move(biases), suppress_tokens, suppress_token_count); + } + + static std::vector skippy_dry_sequence_breaker_storage( +@@ -607,11 +608,6 @@ llama_sampler * skippy_build_sampling_chain( + return nullptr; + } + +- if (skippy_sampling_is_greedy_equivalent(sampling)) { +- llama_sampler_chain_add(sampler, llama_sampler_init_greedy()); +- return sampler; +- } +- + const std::vector logit_biases = + skippy_sampling_logit_biases(session, sampling); + if (!logit_biases.empty()) { +@@ -623,6 +619,10 @@ llama_sampler * skippy_build_sampling_chain( + static_cast(logit_biases.size()), + logit_biases.data())); + } ++ if (skippy_sampling_is_greedy_equivalent(sampling)) { ++ llama_sampler_chain_add(sampler, llama_sampler_init_greedy()); ++ return sampler; ++ } + const uint32_t seed = sampling->seed == 0 ? LLAMA_DEFAULT_SEED : sampling->seed; + const float repeat_penalty = sampling->repeat_penalty == 0.0f + ? 1.0f +diff --git a/src/skippy/sampling_internal.h b/src/skippy/sampling_internal.h +index 9f819704f..4e60b256c 100644 +--- a/src/skippy/sampling_internal.h ++++ b/src/skippy/sampling_internal.h +@@ -8,10 +8,12 @@ + + #include + #include ++#include + + struct llama_context; + struct llama_model; + struct llama_sampler; ++struct llama_vocab; + struct skippy_session; + + void skippy_record_tokens(skippy_session * session, const llama_token * token_ids, size_t token_count); +@@ -29,8 +31,19 @@ enum skippy_status skippy_verify_token_batch( + size_t token_count, + skippy_error ** out_error); + llama_token skippy_greedy_sample_context(const llama_model * model, llama_context * ctx, int32_t index); ++llama_token skippy_greedy_sample_allowed_logits( ++ const float * logits, ++ const llama_vocab * vocab, ++ int32_t n_vocab, ++ const llama_token * suppress_tokens, ++ int32_t suppress_token_count, ++ bool ignore_eog); + llama_token skippy_greedy_sample_ith(skippy_session * session, int32_t index); + llama_sampler * skippy_build_sampling_chain(skippy_session * session, const skippy_sampling_config * sampling); ++std::vector skippy_add_vocab_suppress_token_biases( ++ std::vector biases, ++ const llama_token * suppress_tokens, ++ int32_t suppress_token_count); + void skippy_refresh_backend_sampling(skippy_session * session, const skippy_sampling_config * sampling); + bool skippy_reuse_plain_chat_sampling( + skippy_session * session, +diff --git a/src/skippy/sampling_vocab.cpp b/src/skippy/sampling_vocab.cpp +new file mode 100644 +index 000000000..1242ed105 +--- /dev/null ++++ b/src/skippy/sampling_vocab.cpp +@@ -0,0 +1,63 @@ ++#include "skippy/sampling_internal.h" ++ ++#include "llama.h" ++ ++#include ++#include ++#include ++#include ++#include ++ ++std::vector skippy_add_vocab_suppress_token_biases( ++ std::vector biases, ++ const llama_token * suppress_tokens, ++ int32_t suppress_token_count) { ++ if (suppress_tokens == nullptr || suppress_token_count <= 0) { ++ return biases; ++ } ++ const llama_token * suppress_end = suppress_tokens + suppress_token_count; ++ biases.erase(std::remove_if(biases.begin(), biases.end(), [=](const llama_logit_bias & bias) { ++ return std::find(suppress_tokens, suppress_end, bias.token) != suppress_end; ++ }), biases.end()); ++ biases.reserve(biases.size() + suppress_token_count); ++ for (const llama_token * token = suppress_tokens; token != suppress_end; ++token) { ++ biases.push_back({*token, -INFINITY}); ++ } ++ return biases; ++} ++ ++llama_token skippy_greedy_sample_allowed_logits( ++ const float * logits, ++ const llama_vocab * vocab, ++ int32_t n_vocab, ++ const llama_token * suppress_tokens, ++ int32_t suppress_token_count, ++ bool ignore_eog) { ++ if (logits == nullptr || n_vocab <= 0) { ++ return LLAMA_TOKEN_NULL; ++ } ++ std::vector suppressed; ++ if (suppress_tokens != nullptr && suppress_token_count > 0) { ++ suppressed.resize(static_cast(n_vocab), 0); ++ for (int32_t i = 0; i < suppress_token_count; ++i) { ++ const llama_token token = suppress_tokens[i]; ++ if (token >= 0 && token < n_vocab) { ++ suppressed[static_cast(token)] = 1; ++ } ++ } ++ } ++ ++ llama_token best = LLAMA_TOKEN_NULL; ++ float best_logit = -std::numeric_limits::infinity(); ++ for (llama_token token = 0; token < n_vocab; ++token) { ++ if ((!suppressed.empty() && suppressed[static_cast(token)] != 0) || ++ (ignore_eog && vocab != nullptr && llama_vocab_is_eog(vocab, token))) { ++ continue; ++ } ++ if (best == LLAMA_TOKEN_NULL || logits[token] > best_logit) { ++ best_logit = logits[token]; ++ best = token; ++ } ++ } ++ return best; ++} +diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt +index dc52da145..2b2aae216 100644 +--- a/tests/CMakeLists.txt ++++ b/tests/CMakeLists.txt +@@ -153,6 +153,7 @@ llama_build(test-save-load-state.cpp) + llama_build_and_test(test-skippy-rerank-template.cpp) + + if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) ++ llama_build_and_test(test-skippy-sampling-suppress.cpp) + llama_build_and_test(test-skippy-kv-page-export.cpp) + llama_build_and_test(test-skippy-verify-checkpoint-retirement.cpp) + llama_build_and_test(test-skippy-kv-cells-contiguous.cpp) +diff --git a/tests/test-skippy-sampling-suppress.cpp b/tests/test-skippy-sampling-suppress.cpp +new file mode 100644 +index 000000000..4cf702bef +--- /dev/null ++++ b/tests/test-skippy-sampling-suppress.cpp +@@ -0,0 +1,69 @@ ++#include "../src/skippy/sampling_internal.h" ++ ++#include "llama.h" ++ ++#include ++#include ++#include ++ ++static bool suppressed(const llama_logit_bias & bias) { ++ return std::isinf(bias.bias) && std::signbit(bias.bias); ++} ++ ++int main() { ++ const llama_token suppress_tokens[] = {2, 3}; ++ const std::vector caller_biases = { ++ {2, 100.0f}, ++ {4, -1.0f}, ++ {2, 50.0f}, ++ }; ++ auto merged = skippy_add_vocab_suppress_token_biases( ++ caller_biases, suppress_tokens, 2); ++ if (merged.size() != 3 || merged[0].token != 4 || merged[0].bias != -1.0f || ++ merged[1].token != 2 || !suppressed(merged[1]) || ++ merged[2].token != 3 || !suppressed(merged[2])) { ++ std::fprintf(stderr, "vocab suppress tokens did not override caller logit biases\n"); ++ return 1; ++ } ++ ++ llama_token_data candidates[] = { ++ {0, 1.0f, 0.0f}, ++ {1, 2.0f, 0.0f}, ++ {2, 100.0f, 0.0f}, ++ {3, 50.0f, 0.0f}, ++ {4, 4.0f, 0.0f}, ++ }; ++ llama_token_data_array view = {candidates, 5, -1, false}; ++ llama_sampler * chain = llama_sampler_chain_init(llama_sampler_chain_default_params()); ++ llama_sampler_chain_add(chain, llama_sampler_init_logit_bias(5, merged.size(), merged.data())); ++ llama_sampler_chain_add(chain, llama_sampler_init_greedy()); ++ llama_sampler_apply(chain, &view); ++ llama_sampler_free(chain); ++ if (!std::isinf(candidates[2].logit) || !std::signbit(candidates[2].logit) || ++ !std::isinf(candidates[3].logit) || !std::signbit(candidates[3].logit) || ++ candidates[4].logit != 3.0f || view.selected != 4) { ++ std::fprintf(stderr, "vocab suppression did not reach the sampling chain\n"); ++ return 2; ++ } ++ ++ const float logits[] = {1.0f, 2.0f, 100.0f, 50.0f, 4.0f}; ++ if (skippy_greedy_sample_allowed_logits(logits, nullptr, 5, suppress_tokens, 2, false) != 4 || ++ skippy_greedy_sample_allowed_logits(logits, nullptr, 5, nullptr, 0, false) != 2) { ++ std::fprintf(stderr, "greedy fast path ignored vocab suppression\n"); ++ return 3; ++ } ++ const llama_token all_tokens[] = {0, 1, 2, 3, 4}; ++ const llama_token invalid_token[] = {99}; ++ if (skippy_greedy_sample_allowed_logits(logits, nullptr, 5, all_tokens, 5, false) != LLAMA_TOKEN_NULL || ++ skippy_greedy_sample_allowed_logits(logits, nullptr, 5, invalid_token, 1, false) != 2) { ++ std::fprintf(stderr, "greedy fast path mishandled exhausted or invalid suppression\n"); ++ return 4; ++ } ++ ++ merged = skippy_add_vocab_suppress_token_biases(caller_biases, nullptr, 0); ++ if (merged.size() != caller_biases.size()) { ++ std::fprintf(stderr, "models without suppress tokens lost caller logit biases\n"); ++ return 5; ++ } ++ return 0; ++} +-- +2.54.0 (Apple Git-157) + diff --git a/tools/xtask/data/console_print_allowlist.json b/tools/xtask/data/console_print_allowlist.json index c2824ba4c1..f4bcf4b8e4 100644 --- a/tools/xtask/data/console_print_allowlist.json +++ b/tools/xtask/data/console_print_allowlist.json @@ -4425,7 +4425,7 @@ "macro_name": "eprintln!" }, { - "line": 310, + "line": 270, "macro_name": "eprintln!" } ], @@ -4443,27 +4443,27 @@ ], "crates/skippy-server/src/runtime_state.rs": [ { - "line": 777, + "line": 834, "macro_name": "eprintln!" }, { - "line": 916, + "line": 973, "macro_name": "eprintln!" }, { - "line": 958, + "line": 1015, "macro_name": "eprintln!" }, { - "line": 995, + "line": 1052, "macro_name": "eprintln!" }, { - "line": 999, + "line": 1056, "macro_name": "eprintln!" }, { - "line": 1004, + "line": 1061, "macro_name": "eprintln!" } ] diff --git a/website/src/docs/pages/skippy-api.md b/website/src/docs/pages/skippy-api.md index e35e462a32..9cfee91ed7 100644 --- a/website/src/docs/pages/skippy-api.md +++ b/website/src/docs/pages/skippy-api.md @@ -9,7 +9,7 @@ description: Generated reference for the capability-oriented Skippy C ABI. This reference is generated from the patched llama.cpp public headers. It documents the native C ABI used by Skippy's Rust FFI layer and staged runtime. The ABI is experimental and versioned for lockstep native/Rust builds. -Current generated surface: **15 headers** and **92 exported functions**. +Current generated surface: **16 headers** and **96 exported functions**. ## Quick navigation @@ -173,6 +173,15 @@ Current generated surface: **15 headers** and **92 exported functions**. skippy_parse_chat_response_json +

+ workloads.h4 functions +
+ skippy_model_workload_info_v1 + skippy_session_embed + skippy_session_rerank + skippy_session_encode_prompt +
+
@@ -204,6 +213,7 @@ Capability consumers can include a narrower header: | `include/skippy/stage_plan.h` | Metadata-only construction and inspection of guarded stage plans. | | `include/skippy/state.h` | Moves KV, recurrent, checkpoint, and resident-prefix state. | | `include/skippy/tokenization.h` | Token, detokenization, chat-template, and chat-response helpers. | +| `include/skippy/workloads.h` | Full-model embedding, reranking, and encoder-decoder execution. These operations deliberately reject filtered stage models. Their explicit workload descriptor lets callers fail closed instead of inferring support from a model name or architecture family. | ## ABI conventions @@ -1565,13 +1575,75 @@ SKIPPY_COMMON_API enum skippy_status skippy_parse_chat_response_json( ↩ Back to function index + +### `workloads.h` + + +#### `skippy_model_workload_info_v1` + +Describes the workload implemented by an opened model. + +```cpp +LLAMA_API enum skippy_status skippy_model_workload_info_v1( + const struct skippy_model * model, + struct skippy_workload_info_v1 * out_info, + struct skippy_error ** out_error); +``` + + +#### `skippy_session_embed` + +Computes one normalized embedding for a tokenized input. + +```cpp +LLAMA_API enum skippy_status skippy_session_embed( + struct skippy_session * session, + const llama_token * token_ids, + size_t token_count, + float * output, + size_t output_capacity, + size_t * out_dimensions, + struct skippy_error ** out_error); +``` + + +#### `skippy_session_rerank` + +Computes one scalar relevance score for a query-document pair. + +```cpp +LLAMA_API enum skippy_status skippy_session_rerank( + struct skippy_session * session, + const char * query, + const char * document, + float * out_score, + size_t * out_token_count, + struct skippy_error ** out_error); +``` + + +#### `skippy_session_encode_prompt` + +Encodes an encoder-decoder prompt and returns its first decoder token. + +```cpp +LLAMA_API enum skippy_status skippy_session_encode_prompt( + struct skippy_session * session, + const llama_token * token_ids, + size_t token_count, + llama_token * out_decoder_start_token, + struct skippy_error ** out_error); +``` + +↩ Back to function index + ## Native declarations The headers also define the following enums, structs, opaque handles, and ABI constants: - `activation.h`: `skippy_activation_dtype`, `skippy_activation_layout`, `skippy_activation_boundary_desc`, `skippy_activation_desc`, `SKIPPY_ACTIVATION_BOUNDARY_DESC_VERSION = 1`, `SKIPPY_ACTIVATION_SIDEBAND_TOKEN_IDS = (UINT64_C(1) << 0)`, `SKIPPY_ACTIVATION_FLAG_RWKV7_V_FIRST = (UINT64_C(1) << 0)`, `SKIPPY_ACTIVATION_FLAG_GEMMA3N_ALTUP = (UINT64_C(1) << 1)`, `SKIPPY_ACTIVATION_FLAG_INKLING_MTP_EMBD = (UINT64_C(1) << 2)`, `SKIPPY_ACTIVATION_FLAG_GLM_DSA_TOP_K = (UINT64_C(1) << 3)` -- `common.h`: `skippy_feature`, `skippy_status`, `skippy_error`, `skippy_abi_version`, `SKIPPY_ABI_VERSION_MAJOR = 0`, `SKIPPY_ABI_VERSION_MINOR = 1`, `SKIPPY_ABI_VERSION_PATCH = 53` +- `common.h`: `skippy_feature`, `skippy_status`, `skippy_error`, `skippy_abi_version`, `SKIPPY_ABI_VERSION_MAJOR = 0`, `SKIPPY_ABI_VERSION_MINOR = 1`, `SKIPPY_ABI_VERSION_PATCH = 54` - `devices.h`: `skippy_backend_device_type`, `skippy_backend_device_cap`, `skippy_backend_device` - `events.h`: `skippy_runtime_event_v1`, `skippy_runtime_event_reporter_v1`, `SKIPPY_RUNTIME_EVENT_V1_ABI_VERSION = 1` - `execution.h`: `skippy_iteration_request` @@ -1583,5 +1655,6 @@ The headers also define the following enums, structs, opaque handles, and ABI co - `speculative_decoding.h`: `skippy_ngram_cache`, `skippy_native_mtp_draft`, `SKIPPY_NATIVE_MTP_MAX_DRAFT_TOKENS = 8` - `stage_plan.h`: `skippy_stage_planner`, `skippy_stage_plan`, `skippy_stage_plan_string_ref_v1`, `skippy_stage_planner_tensor_v1`, `skippy_stage_planner_profile_v1`, `skippy_stage_planner_config_v1`, `skippy_stage_plan_value_kind`, `skippy_stage_plan_state_kind`, `skippy_stage_plan_state_access`, `skippy_stage_plan_desc_v1`, `skippy_stage_plan_profile_desc_v1`, `skippy_stage_plan_value_desc_v1`, `skippy_stage_plan_state_desc_v1`, `SKIPPY_STAGE_PLANNER_CONFIG_V1_ABI_VERSION = 1`, `SKIPPY_STAGE_PLANNER_TENSOR_V1_ABI_VERSION = 1`, `SKIPPY_STAGE_PLANNER_PROFILE_V1_ABI_VERSION = 1`, `SKIPPY_STAGE_PLAN_DESC_V1_ABI_VERSION = 1`, `SKIPPY_STAGE_PLAN_PROFILE_DESC_V1_ABI_VERSION = 1`, `SKIPPY_STAGE_PLAN_VALUE_DESC_V1_ABI_VERSION = 1`, `SKIPPY_STAGE_PLAN_STATE_DESC_V1_ABI_VERSION = 1`, `SKIPPY_STAGE_PLAN_MAX_DIMS = 4` - `state.h`: `skippy_kv_page_flag`, `skippy_kv_page_codec`, `skippy_kv_page_component_role`, `skippy_kv_page_component_desc`, `skippy_kv_page_desc` +- `workloads.h`: `skippy_model`, `skippy_session`, `skippy_workload_kind`, `skippy_workload_pooling`, `skippy_workload_info_v1`, `SKIPPY_WORKLOAD_INFO_V1_ABI_VERSION = 1` Source directory: `include/skippy/`. Regenerate this page after changing any public header or exported function. From 19f139dc17583b4cb7c18c461225277914cb51cb Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 12 Sep 2026 11:54:11 -0700 Subject: [PATCH 02/18] test(plugin): publish discard event after its reason --- crates/mesh-native-serving-plugin-host/src/test_support.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/mesh-native-serving-plugin-host/src/test_support.rs b/crates/mesh-native-serving-plugin-host/src/test_support.rs index 9ca77a5966..d145656bf1 100644 --- a/crates/mesh-native-serving-plugin-host/src/test_support.rs +++ b/crates/mesh-native-serving-plugin-host/src/test_support.rs @@ -243,12 +243,14 @@ unsafe extern "C" fn fake_discard_proposal( event: *const abi::ProposalDiscard, ) -> abi::PluginStatus { let state = unsafe { &*instance.cast::() }; - state.events.lock().unwrap().push("discard"); state .discard_reasons .lock() .unwrap() .push(unsafe { (*event).reason }); + // Publish the event only after its observation is complete: tests wait on + // this marker before reading the discard reason from another thread. + state.events.lock().unwrap().push("discard"); thread::sleep(state.report_delay); abi::PluginStatus::OK } From 25ff652dce88390695412c503c43cf01221aadd1 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 12 Sep 2026 13:14:05 -0700 Subject: [PATCH 03/18] fix(openai): gate workloads on each serving target Filter host and passive-client targets before context ranking, cache affinity, reservations, and retries. Keep model discovery independent of descriptor order and prevent cached automatic choices from crossing workload boundaries. Cover mixed advertisements, multipart audio routing, legacy chat compatibility, and the local HTTP rejection path. --- .../src/network/openai/auto_route.rs | 28 +- .../src/network/openai/ingress.rs | 101 ++--- .../src/network/openai/ingress_tests/tests.rs | 2 + .../src/network/openai/mod.rs | 1 + .../src/network/openai/routing_rank.rs | 136 ------- .../src/network/openai/transport.rs | 97 +++-- .../network/openai/transport_route_model.rs | 10 +- .../src/network/openai/transport_tests.rs | 2 + .../network/openai/transport_tests/routing.rs | 6 +- .../transport_tests/workload_routing.rs | 345 ++++++++++++++++++ .../src/network/openai/workload_routing.rs | 219 +++++++++++ .../network/openai/workload_routing/tests.rs | 101 +++++ docs/NON_CHAT_MODELS.md | 15 + 13 files changed, 818 insertions(+), 245 deletions(-) create mode 100644 crates/mesh-llm-host-runtime/src/network/openai/transport_tests/workload_routing.rs create mode 100644 crates/mesh-llm-host-runtime/src/network/openai/workload_routing.rs create mode 100644 crates/mesh-llm-host-runtime/src/network/openai/workload_routing/tests.rs diff --git a/crates/mesh-llm-host-runtime/src/network/openai/auto_route.rs b/crates/mesh-llm-host-runtime/src/network/openai/auto_route.rs index 83d5373b21..3ce7af0c24 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/auto_route.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/auto_route.rs @@ -6,6 +6,7 @@ //! should use that model instead of spending an agent turn on a target we just //! proved unhealthy or too small. +use super::workload_routing; use crate::inference::election; use crate::mesh; use crate::network::affinity::AffinityRouter; @@ -57,11 +58,14 @@ pub(crate) async fn model_has_eligible_target( node: &mesh::Node, model: &str, required_tokens: Option, + request_path: &str, candidates: &[election::InferenceTarget], affinity: &AffinityRouter, ) -> bool { + let candidates = + workload_routing::eligible_targets(node, model, request_path, candidates).await; let context_compatible = - context_compatible_targets(node, model, required_tokens, candidates).await; + context_compatible_targets(node, model, required_tokens, &candidates).await; if !has_routable_candidate(&context_compatible) { return false; } @@ -72,6 +76,7 @@ pub(crate) async fn model_has_eligible_remote_host( node: &mesh::Node, model: &str, required_tokens: Option, + request_path: &str, affinity: &AffinityRouter, ) -> bool { let targets: Vec = node @@ -80,7 +85,15 @@ pub(crate) async fn model_has_eligible_remote_host( .into_iter() .map(election::InferenceTarget::Remote) .collect(); - model_has_eligible_target(node, model, required_tokens, &targets, affinity).await + model_has_eligible_target( + node, + model, + required_tokens, + request_path, + &targets, + affinity, + ) + .await } /// Whether this node itself serves `model`, independent of the target table. @@ -114,12 +127,21 @@ pub(crate) fn pool_for_ready_models<'a>( pub(crate) async fn ready_remote_models<'a>( node: &mesh::Node, required_tokens: Option, + request_path: &str, available: &[router::RoutingCandidate<'a>], affinity: &AffinityRouter, ) -> Vec<&'a str> { let mut ready_models = Vec::new(); for candidate in available { - if model_has_eligible_remote_host(node, candidate.name, required_tokens, affinity).await { + if model_has_eligible_remote_host( + node, + candidate.name, + required_tokens, + request_path, + affinity, + ) + .await + { ready_models.push(candidate.name); } } diff --git a/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs b/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs index ac141f5cf3..66e8b576ef 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs @@ -7,6 +7,9 @@ use crate::network::openai::auto_route; use crate::network::openai::automatic; use crate::network::openai::client_stream::ClientStream; use crate::network::openai::transport as proxy; +use crate::network::openai::workload_routing::{ + self, is_audio_upload_path, model_satisfies_request_workload, request_workload_class, +}; use crate::network::router; use crate::plugin::openai_exchange::{ OpenAiExchangeChannel, OpenAiExchangeDispatchPath, OpenAiExchangeEnvelope, @@ -299,50 +302,33 @@ async fn resolve_auto_routed_model( }, router::classify, ); - let media = body_json.map_or_else( - || router::MediaRequirements { - has_media: is_audio_upload_path(&request.client_path), - needs_vision: false, - needs_audio: is_audio_upload_path(&request.client_path), - }, - router::media_requirements, - ); - let mut available_models = + let media = workload_routing::request_media(&request.client_path, body_json); + let available_models = collect_available_models_for_auto_route(node, targets, plugin_manager).await; - if let Some(workload) = requested_workload { - available_models.retain(|model| { - model_satisfies_request_workload(model, workload, &request.client_path, descriptors) - }); - if available_models.is_empty() { - return AutoRouteResolution::WorkloadUnsupported(workload); - } + let available = workload_routing::routing_candidates( + node, + &available_models, + &request.client_path, + descriptors, + ); + if available.is_empty() + && let Some(workload) = requested_workload + { + return AutoRouteResolution::WorkloadUnsupported(workload); } - let metrics = node.routing_metrics(); - let available: Vec> = available_models - .iter() - .map(|name| { - let caps = proxy::capabilities_for_model(name, descriptors); - let (tps_hint, throughput_samples) = metrics - .tps_for_model(name) - .map(|(t, s)| (Some(t), s)) - .unwrap_or((None, 0)); - router::RoutingCandidate { - name: name.as_str(), - caps, - parameter_count_b: proxy::descriptor_metadata_for_model(name, descriptors) - .and_then(|metadata| metadata.parameter_count_b), - tps_hint, - throughput_samples, - } - }) - .collect(); let Some(available) = router::filter_media_compatible_candidates(&available, &media) else { proxy::release_request_objects(node, &request.request_object_request_ids).await; return AutoRouteResolution::MediaUnsupported; }; - let available = - auto_route_pool_for_ready_models(node, targets, required_tokens, &available, affinity) - .await; + let available = auto_route_pool_for_ready_models( + node, + targets, + required_tokens, + &request.client_path, + &available, + affinity, + ) + .await; let effective_model = router::pick_model_classified(&classification, &available).map(|name| { tracing::info!( @@ -360,44 +346,11 @@ async fn resolve_auto_routed_model( } } -fn is_audio_upload_path(path: &str) -> bool { - matches!( - path.split('?').next().unwrap_or(path), - "/v1/audio/transcriptions" | "/v1/audio/translations" - ) -} - -fn model_satisfies_request_workload( - model: &str, - workload: mesh::ModelWorkloadClass, - path: &str, - descriptors: &[mesh::ServedModelDescriptor], -) -> bool { - if is_audio_upload_path(path) { - proxy::model_satisfies_audio_upload_workload(model, descriptors) - } else { - proxy::model_satisfies_workload_class(model, workload, descriptors) - } -} - -fn request_workload_class(path: &str) -> Option { - match path.split('?').next().unwrap_or(path) { - "/v1/chat/completions" - | "/v1/completions" - | "/v1/responses" - | "/v1/audio/transcriptions" - | "/v1/audio/translations" => Some(mesh::ModelWorkloadClass::CausalGeneration), - "/v1/embeddings" => Some(mesh::ModelWorkloadClass::Embedding), - "/v1/rerank" => Some(mesh::ModelWorkloadClass::Rerank), - "/v1/audio/speech" => Some(mesh::ModelWorkloadClass::SpeechSynthesis), - _ => None, - } -} - async fn auto_route_pool_for_ready_models<'a>( node: &mesh::Node, targets: &election::ModelTargets, required_tokens: Option, + request_path: &str, available: &[router::RoutingCandidate<'a>], affinity: &affinity::AffinityRouter, ) -> Vec> { @@ -408,6 +361,7 @@ async fn auto_route_pool_for_ready_models<'a>( targets, candidate.name, required_tokens, + request_path, affinity, ) .await @@ -423,6 +377,7 @@ async fn auto_route_model_has_ready_ingress_target( targets: &election::ModelTargets, model: &str, required_tokens: Option, + request_path: &str, affinity: &affinity::AffinityRouter, ) -> bool { let local_candidates = targets.candidates(model); @@ -431,6 +386,7 @@ async fn auto_route_model_has_ready_ingress_target( node, model, required_tokens, + request_path, &local_candidates, affinity, ) @@ -448,6 +404,7 @@ async fn auto_route_model_has_ready_ingress_target( node, model, required_tokens, + request_path, &remote_candidates, affinity, ) diff --git a/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/tests.rs b/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/tests.rs index 688e00c587..cd0d08e07f 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/tests.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/tests.rs @@ -105,6 +105,7 @@ async fn phantom_model_is_not_auto_route_eligible() { &targets, "phantom/model:Q4_K_M", None, + "/v1/chat/completions", &affinity, ) .await; @@ -133,6 +134,7 @@ async fn freshly_served_local_model_is_auto_route_eligible() { &targets, "local/fresh-model:Q4_K_M", None, + "/v1/chat/completions", &affinity, ) .await; diff --git a/crates/mesh-llm-host-runtime/src/network/openai/mod.rs b/crates/mesh-llm-host-runtime/src/network/openai/mod.rs index d507faadfc..e32e240f39 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/mod.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/mod.rs @@ -15,3 +15,4 @@ mod response_quality; mod routing_rank; mod tool_call_ids; pub(crate) mod transport; +mod workload_routing; diff --git a/crates/mesh-llm-host-runtime/src/network/openai/routing_rank.rs b/crates/mesh-llm-host-runtime/src/network/openai/routing_rank.rs index cdb5c69435..8506adb6fe 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/routing_rank.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/routing_rank.rs @@ -358,66 +358,6 @@ pub(crate) fn capabilities_for_model( .unwrap_or_else(|| crate::models::installed_model_capabilities(model)) } -pub(crate) fn descriptor_metadata_for_model<'a>( - model: &str, - descriptors: &'a [mesh::ServedModelDescriptor], -) -> Option<&'a mesh::ServedModelMetadata> { - descriptor_for_model(descriptors, model).and_then(|descriptor| descriptor.metadata.as_ref()) -} - -pub(crate) fn workload_class_for_model( - model: &str, - descriptors: &[mesh::ServedModelDescriptor], -) -> Option { - descriptor_metadata_for_model(model, descriptors).and_then(|metadata| metadata.workload_class) -} - -/// Checks the additive workload advertisement without breaking legacy chat -/// routing. An absent field is compatible only with generative endpoints: -/// older nodes predate workload classes and must never be assumed to support a -/// newly introduced non-chat response contract. -pub(crate) fn model_satisfies_workload_class( - model: &str, - requested: mesh::ModelWorkloadClass, - descriptors: &[mesh::ServedModelDescriptor], -) -> bool { - match (requested, workload_class_for_model(model, descriptors)) { - (mesh::ModelWorkloadClass::CausalGeneration, None) => true, - ( - mesh::ModelWorkloadClass::CausalGeneration, - Some( - mesh::ModelWorkloadClass::CausalGeneration - | mesh::ModelWorkloadClass::EncoderDecoder, - ), - ) => true, - (requested, Some(advertised)) => requested == advertised, - (_, None) => false, - } -} - -/// Audio uploads use a newer HTTP contract than chat requests with audio -/// parts. A legacy peer may advertise audio input but lack these endpoints, so -/// neither inferred capabilities nor an absent workload class can opt it in. -pub(crate) fn model_satisfies_audio_upload_workload( - model: &str, - descriptors: &[mesh::ServedModelDescriptor], -) -> bool { - descriptor_for_model(descriptors, model).is_some_and(|descriptor| { - descriptor.capabilities_known - && descriptor.capabilities.supports_audio_runtime() - && matches!( - descriptor - .metadata - .as_ref() - .and_then(|metadata| metadata.workload_class), - Some( - mesh::ModelWorkloadClass::CausalGeneration - | mesh::ModelWorkloadClass::EncoderDecoder - ) - ) - }) -} - #[cfg(test)] mod tests { use super::*; @@ -445,82 +385,6 @@ mod tests { } } - fn descriptor_with_workload( - model_name: &str, - workload_class: mesh::ModelWorkloadClass, - ) -> mesh::ServedModelDescriptor { - mesh::ServedModelDescriptor { - metadata: Some(mesh::ServedModelMetadata { - workload_class: Some(workload_class), - ..Default::default() - }), - ..local_gguf_descriptor(model_name) - } - } - - #[test] - fn legacy_descriptors_are_compatible_only_with_generation_routes() { - let descriptors = vec![local_gguf_descriptor("legacy")]; - - assert!(model_satisfies_workload_class( - "legacy", - mesh::ModelWorkloadClass::CausalGeneration, - &descriptors - )); - assert!(!model_satisfies_workload_class( - "legacy", - mesh::ModelWorkloadClass::Embedding, - &descriptors - )); - assert!(!model_satisfies_workload_class( - "legacy", - mesh::ModelWorkloadClass::SpeechSynthesis, - &descriptors - )); - } - - #[test] - fn workload_routes_require_an_exact_advertised_class() { - let descriptors = vec![ - descriptor_with_workload("embed", mesh::ModelWorkloadClass::Embedding), - descriptor_with_workload("rank", mesh::ModelWorkloadClass::Rerank), - ]; - - assert!(model_satisfies_workload_class( - "embed", - mesh::ModelWorkloadClass::Embedding, - &descriptors - )); - assert!(!model_satisfies_workload_class( - "embed", - mesh::ModelWorkloadClass::Rerank, - &descriptors - )); - assert!(model_satisfies_workload_class( - "rank", - mesh::ModelWorkloadClass::Rerank, - &descriptors - )); - } - - #[test] - fn encoder_decoder_models_can_serve_generation_routes() { - let descriptors = vec![descriptor_with_workload( - "t5", - mesh::ModelWorkloadClass::EncoderDecoder, - )]; - - assert!(model_satisfies_workload_class( - "t5", - mesh::ModelWorkloadClass::CausalGeneration, - &descriptors - )); - assert!(!model_satisfies_workload_class( - "t5", - mesh::ModelWorkloadClass::Embedding, - &descriptors - )); - } #[test] fn test_cached_auto_model_rejects_text_model_for_image_request() { let body = serde_json::json!({ diff --git a/crates/mesh-llm-host-runtime/src/network/openai/transport.rs b/crates/mesh-llm-host-runtime/src/network/openai/transport.rs index 4bd90c830f..b978a9f89a 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/transport.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/transport.rs @@ -15,6 +15,7 @@ use crate::network::affinity::{ use crate::network::openai::auto_route; use crate::network::openai::client_stream::ClientStream; use crate::network::openai::response_quality::ResponseQualityFailure; +use crate::network::openai::workload_routing; use crate::network::router; use std::time::{Duration, Instant}; @@ -29,10 +30,7 @@ pub(crate) use super::response::{ send_400_observed, send_503_observed, send_error_observed, send_json_ok_with_headers, send_json_with_status_and_headers_observed, send_models_list_with_descriptors, }; -pub(crate) use super::routing_rank::{ - capabilities_for_model, descriptor_metadata_for_model, model_satisfies_audio_upload_workload, - model_satisfies_workload_class, request_budget_tokens_from_parts, -}; +pub(crate) use super::routing_rank::{capabilities_for_model, request_budget_tokens_from_parts}; use super::response::{ CacheCostObservation, ResponseRetryPolicy, RouteAttemptLoggingContext, RouteAttemptResult, @@ -188,6 +186,7 @@ pub(crate) fn request_context_budget(request: &BufferedHttpRequest) -> Option), UnsupportedMedia, + UnsupportedWorkload, } enum MeshTargetResolution { @@ -208,6 +207,7 @@ struct MeshRequestPlan { enum MeshRequestFailure { UnsupportedMedia, + UnsupportedWorkload, ModelUnavailable(String), NoHostsAvailable, } @@ -510,7 +510,21 @@ async fn build_mesh_request_plan( AutoModelResolution::UnsupportedMedia => { return Err(MeshRequestFailure::UnsupportedMedia); } + AutoModelResolution::UnsupportedWorkload => { + return Err(MeshRequestFailure::UnsupportedWorkload); + } }; + if let Some(model) = effective_model.as_deref() + && let Some(workload) = workload_routing::request_workload_class(&request.client_path) + && !workload_routing::model_satisfies_request_workload( + model, + workload, + &request.client_path, + &descriptors, + ) + { + return Err(MeshRequestFailure::UnsupportedWorkload); + } rewrite_effective_model(request, effective_model.as_deref()); if is_auto_request { inject_mesh_hooks_flag(&mut request.raw, true); @@ -527,6 +541,16 @@ async fn build_mesh_request_plan( MeshTargetResolution::NoHostsAvailable => return Err(MeshRequestFailure::NoHostsAvailable), }; + let resolved_hosts = if let Some(model) = effective_model.as_deref() { + workload_routing::eligible_remote_hosts(node, model, &request.client_path, &resolved_hosts) + .await + } else { + resolved_hosts + }; + if resolved_hosts.is_empty() { + return Err(MeshRequestFailure::UnsupportedWorkload); + } + let mut prepared = prepare_mesh_targets( request, effective_model.as_deref(), @@ -653,6 +677,15 @@ async fn handle_mesh_request_failure( ) { let mut tcp_stream = Some(tcp_stream); match failure { + MeshRequestFailure::UnsupportedWorkload => { + let _ = send_error_observed( + tcp_stream.take().unwrap(), + 422, + "no serving target advertises support for the requested workload endpoint", + route_observer, + ) + .await; + } MeshRequestFailure::UnsupportedMedia => { let _ = send_error_observed( tcp_stream.take().unwrap(), @@ -1017,6 +1050,9 @@ fn terminal_outcome_for_mesh_request_failure( failure: &MeshRequestFailure, ) -> crate::logging::TerminalOutcome { match failure { + MeshRequestFailure::UnsupportedWorkload => { + crate::logging::TerminalOutcome::Rejected(Some("unsupported_workload".into())) + } MeshRequestFailure::UnsupportedMedia => { crate::logging::TerminalOutcome::Rejected(Some("unsupported_media".into())) } @@ -1161,34 +1197,30 @@ async fn resolve_auto_model_request(args: AutoModelRequestArgs<'_>) -> AutoModel return AutoModelResolution::Model(None); } request.ensure_body_json(); - let Some(body_json) = request.body_json.as_ref() else { + if request.body_json.is_none() && !workload_routing::is_audio_upload_path(&request.client_path) + { return AutoModelResolution::Model(None); - }; - let media = router::media_requirements(body_json); - // Build candidates with observed throughput so pick_model_classified - // can weight by locally-measured tok/s where samples exist. - let routing_metrics = node.routing_metrics(); - let with_caps: Vec> = served - .iter() - .map(|name| { - let caps = capabilities_for_model(name, descriptors); - let (tps_hint, throughput_samples) = routing_metrics - .tps_for_model(name) - .map(|(tps, samples)| (Some(tps), samples)) - .unwrap_or((None, 0)); - router::RoutingCandidate { - name: name.as_str(), - caps, - parameter_count_b: descriptor_metadata_for_model(name, descriptors) - .and_then(|metadata| metadata.parameter_count_b), - tps_hint, - throughput_samples, - } - }) - .collect(); + } + let empty_body = serde_json::Value::Null; + let body_json = request.body_json.as_ref().unwrap_or(&empty_body); + let media = workload_routing::request_media(&request.client_path, request.body_json.as_ref()); + let with_caps = + workload_routing::routing_candidates(node, served, &request.client_path, descriptors); + if with_caps.is_empty() + && workload_routing::request_workload_class(&request.client_path).is_some() + { + return AutoModelResolution::UnsupportedWorkload; + } let available = router::filter_media_compatible_candidates(&with_caps, &media); let ready_models = if let Some(available) = available.as_ref() { - auto_route::ready_remote_models(node, required_tokens, available, affinity).await + auto_route::ready_remote_models( + node, + required_tokens, + &request.client_path, + available, + affinity, + ) + .await } else { Vec::new() }; @@ -1202,7 +1234,12 @@ async fn resolve_auto_model_request(args: AutoModelRequestArgs<'_>) -> AutoModel ) .await { - return AutoModelResolution::Model(Some(model)); + if with_caps.iter().any(|candidate| candidate.name == model) { + return AutoModelResolution::Model(Some(model)); + } + if let Some(key) = auto_session_key { + affinity.forget_auto_model(key); + } } let Some(available) = available else { diff --git a/crates/mesh-llm-host-runtime/src/network/openai/transport_route_model.rs b/crates/mesh-llm-host-runtime/src/network/openai/transport_route_model.rs index adbdaae522..635714bc04 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/transport_route_model.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/transport_route_model.rs @@ -95,8 +95,14 @@ async fn route_model_request_inner(args: RouteModelRequestArgs<'_>) -> RouteDisp } = args; let route_started = Instant::now(); let mut tcp_stream = tcp_stream; - let ranked = - rank_targets_by_context(&node, model, required_tokens, &targets.candidates(model)).await; + let candidates = super::super::workload_routing::eligible_targets( + &node, + model, + &request.client_path, + &targets.candidates(model), + ) + .await; + let ranked = rank_targets_by_context(&node, model, required_tokens, &candidates).await; let ordered_candidates = affinity.route_eligible_candidates(model, &ranked.ordered); if ordered_candidates.is_empty() { record_route_model_unavailable(&node, model, 0); diff --git a/crates/mesh-llm-host-runtime/src/network/openai/transport_tests.rs b/crates/mesh-llm-host-runtime/src/network/openai/transport_tests.rs index eba00caf8a..ee45f57357 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/transport_tests.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/transport_tests.rs @@ -143,3 +143,5 @@ mod durable_artifacts; mod lifecycle; #[path = "transport_tests/routing.rs"] mod routing; +#[path = "transport_tests/workload_routing.rs"] +mod workload_routing; diff --git a/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/routing.rs b/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/routing.rs index a1b57faf54..0fe15ccf4d 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/routing.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/routing.rs @@ -65,7 +65,7 @@ impl crate::network::metrics::RoutingTelemetrySink for PromptShapeSink { } } -fn test_peer_serving_model(peer_id: iroh::EndpointId, model: &str) -> mesh::PeerInfo { +pub(super) fn test_peer_serving_model(peer_id: iroh::EndpointId, model: &str) -> mesh::PeerInfo { mesh::PeerInfo { id: peer_id, addr: iroh::EndpointAddr { @@ -500,7 +500,9 @@ async fn cached_auto_model_stays_sticky_when_no_ready_remote_model_exists() -> R router::RoutingCandidate::unscored(cached_model, caps), router::RoutingCandidate::unscored(alternate_model, caps), ]; - let ready_models = auto_route::ready_remote_models(&node, None, &available, &affinity).await; + let ready_models = + auto_route::ready_remote_models(&node, None, "/v1/chat/completions", &available, &affinity) + .await; assert!(ready_models.is_empty()); let cached = lookup_cached_auto_model( diff --git a/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/workload_routing.rs b/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/workload_routing.rs new file mode 100644 index 0000000000..58a6486c59 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/workload_routing.rs @@ -0,0 +1,345 @@ +use super::routing::test_peer_serving_model; +use super::*; +use crate::mesh::ModelWorkloadClass; +use crate::models::{CapabilityLevel, ModelCapabilities}; +use crate::network::openai::workload_routing; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +const MODEL: &str = "shared-workload-model"; + +fn descriptor(class: Option, audio: bool) -> mesh::ServedModelDescriptor { + mesh::ServedModelDescriptor { + identity: mesh::ServedModelIdentity { + model_name: MODEL.into(), + ..Default::default() + }, + metadata: Some(mesh::ServedModelMetadata { + workload_class: class, + ..Default::default() + }), + capabilities_known: true, + capabilities: ModelCapabilities { + audio: if audio { + CapabilityLevel::Supported + } else { + CapabilityLevel::None + }, + multimodal: audio, + ..Default::default() + }, + ..Default::default() + } +} + +async fn peer( + node: &mesh::Node, + class: Option, + audio: bool, +) -> iroh::EndpointId { + let id = iroh::SecretKey::generate().public(); + let mut peer = test_peer_serving_model(id, MODEL); + peer.served_model_descriptors = vec![descriptor(class, audio)]; + node.insert_test_peer(peer).await; + id +} + +fn request(path: &str, model: &str) -> BufferedHttpRequest { + let body = serde_json::json!({ + "model": model, + "input": "workload routing regression", + "user": "workload-session", + "prompt_cache_key": "workload-cache", + }); + let bytes = serde_json::to_vec(&body).expect("serialize body"); + BufferedHttpRequest { + raw: format!("POST {path} HTTP/1.1\r\nHost: localhost\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{}", bytes.len(), body).into_bytes(), + method: "POST".into(), + path: path.into(), + client_path: path.into(), + request_id: RequestId::default(), + body_json: Some(body), + body_json_attempted: true, + body_len_bytes: bytes.len(), + body_bytes: Some(bytes), + completion_tokens: None, + stream: None, + model_name: Some(model.into()), + request_object_request_ids: vec![], + response_adapter: ResponseAdapter::None, + correlation_id: None, + } +} + +#[tokio::test] +async fn non_chat_targets_require_their_own_workload_advertisement() { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Client) + .await + .expect("node"); + let legacy = peer(&node, None, true).await; + for (path, class, audio) in [ + ("/v1/embeddings", ModelWorkloadClass::Embedding, false), + ("/v1/rerank", ModelWorkloadClass::Rerank, false), + ( + "/v1/audio/speech", + ModelWorkloadClass::SpeechSynthesis, + false, + ), + ( + "/v1/audio/transcriptions", + ModelWorkloadClass::CausalGeneration, + true, + ), + ( + "/v1/audio/translations?trace=1", + ModelWorkloadClass::CausalGeneration, + true, + ), + ] { + let capable = peer(&node, Some(class), audio).await; + let candidates = vec![ + election::InferenceTarget::Remote(legacy), + election::InferenceTarget::Remote(capable), + ]; + assert_eq!( + workload_routing::eligible_targets(&node, MODEL, path, &candidates).await, + vec![election::InferenceTarget::Remote(capable)], + "{path} must not inherit another peer's capability", + ); + } +} + +#[tokio::test] +async fn local_target_cannot_inherit_a_remote_workload_or_vice_versa() { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Worker) + .await + .expect("node"); + let remote = peer(&node, Some(ModelWorkloadClass::Embedding), false).await; + node.set_served_model_descriptors(vec![descriptor(None, false)]) + .await; + let local = election::InferenceTarget::Local(9337); + let remote_target = election::InferenceTarget::Remote(remote); + let candidates = vec![local.clone(), remote_target.clone()]; + assert_eq!( + workload_routing::eligible_targets(&node, MODEL, "/v1/embeddings", &candidates).await, + vec![remote_target], + ); + + node.set_served_model_descriptors(vec![descriptor(Some(ModelWorkloadClass::Embedding), false)]) + .await; + let mut legacy_peer = test_peer_serving_model(remote, MODEL); + legacy_peer.served_model_descriptors = vec![descriptor(None, false)]; + node.insert_test_peer(legacy_peer).await; + assert_eq!( + workload_routing::eligible_targets(&node, MODEL, "/v1/embeddings", &candidates).await, + vec![local], + ); +} + +#[tokio::test] +async fn legacy_generation_and_control_routes_remain_eligible() { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Client) + .await + .expect("node"); + let legacy = peer(&node, None, true).await; + let targets = vec![election::InferenceTarget::Remote(legacy)]; + for path in [ + "/v1/chat/completions", + "/v1/completions", + "/v1/responses", + "/tokenize", + ] { + assert_eq!( + workload_routing::eligible_targets(&node, MODEL, path, &targets).await, + targets + ); + } + node.remove_test_peer(legacy).await; + assert!( + workload_routing::eligible_targets(&node, MODEL, "/v1/embeddings", &targets) + .await + .is_empty() + ); +} + +#[tokio::test] +async fn passive_plan_excludes_legacy_hosts_before_affinity_and_reservation() { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Client) + .await + .expect("node"); + let legacy = peer(&node, None, false).await; + let capable = peer(&node, Some(ModelWorkloadClass::Embedding), false).await; + let affinity = AffinityRouter::new(); + let mut request = request("/v1/embeddings", MODEL); + let prefix = + crate::network::affinity::cache_prefix_hash(request.body_json.as_ref()).expect("prefix"); + affinity.remember_cache_lease_if_epoch( + MODEL, + prefix, + &election::InferenceTarget::Remote(legacy), + affinity.cache_lease_epoch(), + ); + let plan = build_mesh_request_plan(&node, &mut request, false, &affinity) + .await + .unwrap_or_else(|_| panic!("capable peer must remain routable")); + assert_eq!(plan.target_hosts, vec![capable]); + assert_eq!(plan.equivalent_hosts, 1); + let (hosts, _reservation) = reserve_mesh_request_target(&plan, &affinity); + assert_eq!(hosts, vec![capable]); +} + +#[tokio::test] +async fn passive_plan_rejects_unknown_workloads_instead_of_forwarding_them() { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Client) + .await + .expect("node"); + peer(&node, None, true).await; + let affinity = AffinityRouter::new(); + for path in ["/v1/embeddings", "/v1/audio/transcriptions"] { + let mut request = request(path, MODEL); + assert!(matches!( + build_mesh_request_plan(&node, &mut request, false, &affinity).await, + Err(MeshRequestFailure::UnsupportedWorkload), + )); + } +} + +#[tokio::test] +async fn passive_auto_audio_uses_the_capable_descriptor_and_rewrites_multipart() { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Client) + .await + .expect("node"); + peer(&node, None, false).await; + let capable = peer(&node, Some(ModelWorkloadClass::CausalGeneration), true).await; + // A local legacy descriptor deliberately precedes all current peer metadata. + node.set_served_model_descriptors(vec![descriptor(None, false)]) + .await; + let path = "/v1/audio/transcriptions"; + let mut request = request(path, "auto"); + let body = b"--fixture\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\nauto\r\n--fixture\r\nContent-Disposition: form-data; name=\"file\"; filename=\"audio.wav\"\r\n\r\nRIFF\r\n--fixture--\r\n".to_vec(); + request.raw = format!("POST {path} HTTP/1.1\r\nHost: localhost\r\nContent-Type: multipart/form-data; boundary=fixture\r\nContent-Length: {}\r\n\r\n", body.len()).into_bytes(); + request.raw.extend_from_slice(&body); + request.body_len_bytes = body.len(); + request.body_bytes = Some(body); + request.body_json = None; + let plan = build_mesh_request_plan(&node, &mut request, false, &AffinityRouter::new()) + .await + .unwrap_or_else(|_| panic!("current audio peer must be selected")); + assert_eq!(plan.effective_model.as_deref(), Some(MODEL)); + assert_eq!(plan.target_hosts, vec![capable]); + assert!(String::from_utf8_lossy(&request.raw).contains(&format!("\r\n\r\n{MODEL}\r\n"))); + assert!(String::from_utf8_lossy(&request.raw).contains("\r\n\r\nRIFF\r\n")); +} + +#[tokio::test] +async fn passive_auto_model_cache_cannot_cross_workload_boundaries() { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Client) + .await + .expect("node"); + let capable = peer(&node, Some(ModelWorkloadClass::Embedding), false).await; + let chat_id = iroh::SecretKey::generate().public(); + node.insert_test_peer(test_peer_serving_model(chat_id, "legacy-chat")) + .await; + let affinity = AffinityRouter::new(); + let mut request = request("/v1/embeddings", "auto"); + let key = auto_session_key_for_request(&mut request, true).expect("explicit cache key"); + affinity.remember_auto_model(key, "legacy-chat"); + // No healthy compatible alternative: the availability fallback must still + // stay inside the requested workload, never restore the cached chat model. + affinity.record_target_outcome( + Some(MODEL), + &election::InferenceTarget::Remote(capable), + TargetHealthOutcome::Unavailable, + ); + let plan = build_mesh_request_plan(&node, &mut request, false, &affinity) + .await + .unwrap_or_else(|_| panic!("embedding fallback remains available")); + assert_eq!(plan.effective_model.as_deref(), Some(MODEL)); + assert_eq!(plan.target_hosts, vec![capable]); + assert_eq!(affinity.lookup_auto_model(key).as_deref(), Some(MODEL)); +} + +#[tokio::test] +async fn audio_upload_capabilities_must_belong_to_one_descriptor_on_the_target() { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Client) + .await + .expect("node"); + let id = iroh::SecretKey::generate().public(); + let mut mixed = test_peer_serving_model(id, MODEL); + mixed.served_model_descriptors = vec![ + descriptor(None, true), + descriptor(Some(ModelWorkloadClass::CausalGeneration), false), + ]; + node.insert_test_peer(mixed).await; + let candidates = vec![election::InferenceTarget::Remote(id)]; + assert!( + workload_routing::eligible_targets(&node, MODEL, "/v1/audio/transcriptions", &candidates) + .await + .is_empty() + ); +} + +#[tokio::test] +async fn host_dispatch_rejects_local_legacy_target_despite_capable_remote_metadata() { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Worker) + .await + .expect("node"); + peer(&node, Some(ModelWorkloadClass::Embedding), false).await; + node.set_served_model_descriptors(vec![descriptor(None, false)]) + .await; + let backend = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("backend"); + let backend_port = backend.local_addr().expect("address").port(); + let backend_task = tokio::spawn(async move { + if let Ok(Ok((mut stream, _))) = + tokio::time::timeout(Duration::from_secs(5), backend.accept()).await + { + let mut request_bytes = [0; 4096]; + let _ = stream.read(&mut request_bytes).await; + let _ = stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}") + .await; + } + }); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("client listener"); + let (client, server) = tokio::join!( + tokio::net::TcpStream::connect(listener.local_addr().expect("address")), + listener.accept() + ); + let mut client = client.expect("client"); + let (server, _) = server.expect("server"); + let mut targets = election::ModelTargets::default(); + targets.targets.insert( + MODEL.into(), + vec![election::InferenceTarget::Local(backend_port)], + ); + let request = request("/v1/embeddings", MODEL); + let affinity = AffinityRouter::new(); + let outcome = tokio::time::timeout( + Duration::from_secs(2), + route_model_request( + node, + server.into(), + &targets, + MODEL, + &request, + RouteModelRequestContext { + required_tokens: None, + affinity: &affinity, + route_observer: OpenAiRouteObserver::default(), + }, + ), + ) + .await; + backend_task.abort(); + let _ = backend_task.await; + assert!(matches!(outcome, Ok(RouteDispatchOutcome::Responded(503)))); + let mut response = String::new(); + client + .read_to_string(&mut response) + .await + .expect("response"); + assert!(response.starts_with("HTTP/1.1 503")); +} diff --git a/crates/mesh-llm-host-runtime/src/network/openai/workload_routing.rs b/crates/mesh-llm-host-runtime/src/network/openai/workload_routing.rs new file mode 100644 index 0000000000..29f482bf48 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/network/openai/workload_routing.rs @@ -0,0 +1,219 @@ +//! Workload admission at the model and individual serving-target boundaries. +//! +//! Model-level admission is existential: any matching advertisement can make a +//! model discoverable. It never grants another peer that model's capabilities. +//! Both host and passive-client dispatch filter their concrete targets before +//! context ranking, affinity, reservations, and retry selection. + +use crate::inference::election::InferenceTarget; +use crate::mesh::{self, ModelWorkloadClass, ServedModelDescriptor}; + +#[cfg(test)] +mod tests; + +pub(super) fn is_audio_upload_path(path: &str) -> bool { + matches!( + path.split('?').next().unwrap_or(path), + "/v1/audio/transcriptions" | "/v1/audio/translations" + ) +} + +pub(super) fn request_workload_class(path: &str) -> Option { + match path.split('?').next().unwrap_or(path) { + "/v1/chat/completions" + | "/v1/completions" + | "/v1/responses" + | "/v1/audio/transcriptions" + | "/v1/audio/translations" => Some(ModelWorkloadClass::CausalGeneration), + "/v1/embeddings" => Some(ModelWorkloadClass::Embedding), + "/v1/rerank" => Some(ModelWorkloadClass::Rerank), + "/v1/audio/speech" => Some(ModelWorkloadClass::SpeechSynthesis), + _ => None, + } +} + +fn class_is_compatible( + requested: ModelWorkloadClass, + advertised: Option, +) -> bool { + match (requested, advertised) { + (ModelWorkloadClass::CausalGeneration, None) => true, + ( + ModelWorkloadClass::CausalGeneration, + Some(ModelWorkloadClass::CausalGeneration | ModelWorkloadClass::EncoderDecoder), + ) => true, + (requested, Some(advertised)) => requested == advertised, + (_, None) => false, + } +} + +pub(super) fn model_satisfies_workload_class( + model: &str, + requested: ModelWorkloadClass, + descriptors: &[ServedModelDescriptor], +) -> bool { + let mut matching = descriptors + .iter() + .filter(|descriptor| descriptor.identity.model_name == model) + .peekable(); + if matching.peek().is_none() { + return class_is_compatible(requested, None); + } + matching.any(|descriptor| { + class_is_compatible( + requested, + descriptor + .metadata + .as_ref() + .and_then(|metadata| metadata.workload_class), + ) + }) +} + +fn descriptor_supports_audio_upload(descriptor: &ServedModelDescriptor) -> bool { + descriptor.capabilities_known + && descriptor.capabilities.supports_audio_runtime() + && matches!( + descriptor + .metadata + .as_ref() + .and_then(|metadata| metadata.workload_class), + Some(ModelWorkloadClass::CausalGeneration | ModelWorkloadClass::EncoderDecoder) + ) +} + +pub(super) fn model_satisfies_request_workload( + model: &str, + workload: ModelWorkloadClass, + path: &str, + descriptors: &[ServedModelDescriptor], +) -> bool { + if is_audio_upload_path(path) { + descriptors.iter().any(|descriptor| { + descriptor.identity.model_name == model && descriptor_supports_audio_upload(descriptor) + }) + } else { + model_satisfies_workload_class(model, workload, descriptors) + } +} + +pub(super) fn descriptor_for_request<'a>( + model: &str, + path: &str, + descriptors: &'a [ServedModelDescriptor], +) -> Option<&'a ServedModelDescriptor> { + descriptors.iter().find(|descriptor| { + descriptor.identity.model_name == model + && request_workload_class(path).is_none_or(|workload| { + model_satisfies_request_workload( + model, + workload, + path, + std::slice::from_ref(*descriptor), + ) + }) + }) +} + +pub(super) fn request_media( + path: &str, + body: Option<&serde_json::Value>, +) -> crate::network::router::MediaRequirements { + if is_audio_upload_path(path) { + crate::network::router::MediaRequirements { + has_media: true, + needs_audio: true, + needs_vision: false, + } + } else { + body.map_or_else(Default::default, crate::network::router::media_requirements) + } +} + +/// Pick metadata from a descriptor that supports this request, not whichever +/// peer happened to gossip the model name first. +pub(super) fn routing_candidates<'a>( + node: &mesh::Node, + models: &'a [String], + path: &str, + descriptors: &[ServedModelDescriptor], +) -> Vec> { + let metrics = node.routing_metrics(); + models + .iter() + .filter(|model| { + request_workload_class(path).is_none_or(|workload| { + model_satisfies_request_workload(model, workload, path, descriptors) + }) + }) + .map(|model| { + let descriptor = descriptor_for_request(model, path, descriptors); + let caps = descriptor.map_or_else( + || super::routing_rank::capabilities_for_model(model, descriptors), + |descriptor| descriptor.capabilities, + ); + let (tps_hint, throughput_samples) = metrics + .tps_for_model(model) + .map(|(tps, samples)| (Some(tps), samples)) + .unwrap_or((None, 0)); + crate::network::router::RoutingCandidate { + name: model, + caps, + parameter_count_b: descriptor + .and_then(|descriptor| descriptor.metadata.as_ref()) + .and_then(|metadata| metadata.parameter_count_b), + tps_hint, + throughput_samples, + } + }) + .collect() +} + +pub(super) async fn eligible_targets( + node: &mesh::Node, + model: &str, + path: &str, + candidates: &[InferenceTarget], +) -> Vec { + let Some(workload) = request_workload_class(path) else { + return candidates.to_vec(); + }; + let local = node.served_model_descriptors().await; + let state = node.state.lock().await; + candidates + .iter() + .filter(|target| { + let descriptors = match target { + InferenceTarget::Local(_) => local.as_slice(), + InferenceTarget::Remote(peer_id) => state + .peers + .get(peer_id) + .map_or(&[][..], |peer| peer.served_model_descriptors.as_slice()), + InferenceTarget::None => return false, + }; + model_satisfies_request_workload(model, workload, path, descriptors) + }) + .cloned() + .collect() +} + +pub(super) async fn eligible_remote_hosts( + node: &mesh::Node, + model: &str, + path: &str, + hosts: &[iroh::EndpointId], +) -> Vec { + let candidates = hosts + .iter() + .copied() + .map(InferenceTarget::Remote) + .collect::>(); + eligible_targets(node, model, path, &candidates) + .await + .into_iter() + .filter_map(|target| match target { + InferenceTarget::Remote(peer) => Some(peer), + _ => None, + }) + .collect() +} diff --git a/crates/mesh-llm-host-runtime/src/network/openai/workload_routing/tests.rs b/crates/mesh-llm-host-runtime/src/network/openai/workload_routing/tests.rs new file mode 100644 index 0000000000..8cb3ff1fad --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/network/openai/workload_routing/tests.rs @@ -0,0 +1,101 @@ +use super::*; + +fn local_gguf_descriptor(model_name: &str) -> ServedModelDescriptor { + ServedModelDescriptor { + identity: mesh::ServedModelIdentity { + model_name: model_name.into(), + ..Default::default() + }, + ..Default::default() + } +} + +fn descriptor_with_workload( + model_name: &str, + workload_class: mesh::ModelWorkloadClass, +) -> mesh::ServedModelDescriptor { + mesh::ServedModelDescriptor { + metadata: Some(mesh::ServedModelMetadata { + workload_class: Some(workload_class), + ..Default::default() + }), + ..local_gguf_descriptor(model_name) + } +} + +#[test] +fn legacy_descriptors_are_compatible_only_with_generation_routes() { + let descriptors = vec![local_gguf_descriptor("legacy")]; + + assert!(model_satisfies_workload_class( + "legacy", + mesh::ModelWorkloadClass::CausalGeneration, + &descriptors + )); + assert!(!model_satisfies_workload_class( + "legacy", + mesh::ModelWorkloadClass::Embedding, + &descriptors + )); + assert!(!model_satisfies_workload_class( + "legacy", + mesh::ModelWorkloadClass::SpeechSynthesis, + &descriptors + )); +} + +#[test] +fn workload_routes_require_an_exact_advertised_class() { + let descriptors = vec![ + descriptor_with_workload("embed", mesh::ModelWorkloadClass::Embedding), + descriptor_with_workload("rank", mesh::ModelWorkloadClass::Rerank), + ]; + + assert!(model_satisfies_workload_class( + "embed", + mesh::ModelWorkloadClass::Embedding, + &descriptors + )); + assert!(!model_satisfies_workload_class( + "embed", + mesh::ModelWorkloadClass::Rerank, + &descriptors + )); + assert!(model_satisfies_workload_class( + "rank", + mesh::ModelWorkloadClass::Rerank, + &descriptors + )); +} + +#[test] +fn workload_metadata_is_independent_of_descriptor_order() { + let legacy = local_gguf_descriptor("shared-model"); + let current = descriptor_with_workload("shared-model", mesh::ModelWorkloadClass::Embedding); + for descriptors in [vec![legacy.clone(), current.clone()], vec![current, legacy]] { + assert!(model_satisfies_workload_class( + "shared-model", + mesh::ModelWorkloadClass::Embedding, + &descriptors, + )); + } +} + +#[test] +fn encoder_decoder_models_can_serve_generation_routes() { + let descriptors = vec![descriptor_with_workload( + "t5", + mesh::ModelWorkloadClass::EncoderDecoder, + )]; + + assert!(model_satisfies_workload_class( + "t5", + mesh::ModelWorkloadClass::CausalGeneration, + &descriptors + )); + assert!(!model_satisfies_workload_class( + "t5", + mesh::ModelWorkloadClass::Embedding, + &descriptors + )); +} diff --git a/docs/NON_CHAT_MODELS.md b/docs/NON_CHAT_MODELS.md index 05d6cc2a3f..a1ebc027ff 100644 --- a/docs/NON_CHAT_MODELS.md +++ b/docs/NON_CHAT_MODELS.md @@ -22,6 +22,21 @@ a structured `unsupported` error. OCR and speech recognition use causal trunks, but their projector remains local to the trunk; they are not claims of a distributed projector implementation. +## Mixed-version routing + +A model name can be advertised by several nodes running different versions. +Model discovery accepts any compatible advertisement, but each serving target +must independently advertise the workload required by the endpoint. The host +router and passive-client proxy exclude incompatible targets before context +ranking, cache affinity, reservation spreading, and retries. One current peer +does not grant its endpoints to a legacy peer serving the same model name. + +Absent workload metadata remains compatible with ordinary generation requests, +not with embedding, rerank, speech synthesis, or audio-upload endpoints. Audio +uploads additionally require runtime-verified audio support in the same target's +model descriptor. Unsupported targets are not restored by an availability +fallback or a cached automatic model choice. + ## Embeddings The request accepts a string, an array of strings, a token array, or an array of From 0bbc85bd9294926d876fc35a62144ec64f589e30 Mon Sep 17 00:00:00 2001 From: IvGolovach <20299097+IvGolovach@users.noreply.github.com> Date: Sat, 12 Sep 2026 16:09:38 -0700 Subject: [PATCH 04/18] fix(openai): preserve workload routing and multipart contracts --- crates/mesh-llm-host-runtime/src/mesh/mod.rs | 2 +- .../src/mesh/model_identity.rs | 53 ++++++ .../src/mesh/peer_state.rs | 38 ----- .../src/network/openai/ingress.rs | 7 +- .../openai/ingress_tests/audio_workloads.rs | 116 +++++++++++++ .../openai/ingress_tests/automatic_routing.rs | 29 ++++ .../openai/moa_gateway/context_selection.rs | 1 + .../src/network/openai/moa_gateway/mod.rs | 11 +- .../src/network/openai/moa_gateway/pool.rs | 27 ++- .../network/openai/moa_gateway/self_fill.rs | 2 + .../openai/moa_gateway/workload_admission.rs | 82 +++++++++ .../moa_gateway/workload_admission/tests.rs | 140 +++++++++++++++ .../src/network/openai/request_parse.rs | 129 +------------- .../openai/request_parse/body_rewrite.rs | 159 ++++++++++++++++++ .../request_parse/body_rewrite/tests.rs | 117 +++++++++++++ .../src/network/openai/request_parse_tests.rs | 2 +- .../src/network/openai/transport.rs | 4 +- .../network/openai/transport_route_model.rs | 9 +- .../transport_tests/workload_routing.rs | 60 ++++++- .../src/network/openai/workload_routing.rs | 49 +++++- 20 files changed, 850 insertions(+), 187 deletions(-) create mode 100644 crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/audio_workloads.rs create mode 100644 crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/workload_admission.rs create mode 100644 crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/workload_admission/tests.rs create mode 100644 crates/mesh-llm-host-runtime/src/network/openai/request_parse/body_rewrite.rs create mode 100644 crates/mesh-llm-host-runtime/src/network/openai/request_parse/body_rewrite/tests.rs diff --git a/crates/mesh-llm-host-runtime/src/mesh/mod.rs b/crates/mesh-llm-host-runtime/src/mesh/mod.rs index 4207e70b1c..9f0bff5cc3 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/mod.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/mod.rs @@ -90,7 +90,7 @@ mod heartbeat; mod host_role_claims; mod identity_persistence; mod lan_bootstrap; -mod model_identity; +pub(crate) mod model_identity; mod node; mod node_identity; mod node_requirements; diff --git a/crates/mesh-llm-host-runtime/src/mesh/model_identity.rs b/crates/mesh-llm-host-runtime/src/mesh/model_identity.rs index 1a9e2a2177..36f31ac09f 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/model_identity.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/model_identity.rs @@ -1,3 +1,5 @@ +//! Public routing names derived from served identity and the loaded catalog. + use super::*; pub(crate) fn infer_remote_served_descriptors( @@ -209,6 +211,57 @@ pub(crate) fn identity_hash_for(input: &str) -> String { hex::encode(hasher.finalize()) } +pub(crate) fn public_model_id_from_identity(identity: &ServedModelIdentity) -> Option { + match identity.source_kind { + ModelSourceKind::HuggingFace => identity + .repository + .as_deref() + .map(|repo| { + let selector = identity + .artifact + .as_deref() + .and_then(model_ref::quant_selector_from_gguf_file) + .or_else(|| identity.artifact.clone()); + model_ref::format_model_ref(repo, identity.revision.as_deref(), selector.as_deref()) + }) + .or_else(|| { + identity + .canonical_ref + .as_deref() + .and_then(|model_ref| model_ref::ModelRef::parse(model_ref).ok()) + .map(|model_ref| model_ref.display_id()) + }), + ModelSourceKind::Catalog => identity + .canonical_ref + .as_deref() + .and_then(|model_ref| model_ref::ModelRef::parse(model_ref).ok()) + .map(|model_ref| model_ref.display_id()), + ModelSourceKind::LocalGguf | ModelSourceKind::DirectUrl | ModelSourceKind::Unknown => None, + } +} + +pub(crate) fn canonical_demand_model_ref(model: &str) -> String { + if let Ok(model_ref) = model_ref::ModelRef::parse(model) { + return model_ref.display_id(); + } + crate::models::find_loaded_remote_catalog_model_exact(model) + .map(|remote_model| crate::models::remote_catalog_model_ref(&remote_model)) + .unwrap_or_else(|| model.to_string()) +} + +/// Match exactly the same public alias that peer HTTP discovery advertises. +/// Do not infer identity from similar basenames or borrow another peer's facts. +pub(crate) fn descriptor_matches_routable_name( + descriptor: &ServedModelDescriptor, + name: &str, +) -> bool { + let identity = &descriptor.identity; + identity.model_name == name + || public_model_id_from_identity(identity) + .unwrap_or_else(|| canonical_demand_model_ref(&identity.model_name)) + == name +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/mesh-llm-host-runtime/src/mesh/peer_state.rs b/crates/mesh-llm-host-runtime/src/mesh/peer_state.rs index a07b27caa0..e4d39f5c69 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/peer_state.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/peer_state.rs @@ -492,44 +492,6 @@ impl PeerInfo { } } -pub(crate) fn public_model_id_from_identity(identity: &ServedModelIdentity) -> Option { - match identity.source_kind { - ModelSourceKind::HuggingFace => identity - .repository - .as_deref() - .map(|repo| { - let selector = identity - .artifact - .as_deref() - .and_then(model_ref::quant_selector_from_gguf_file) - .or_else(|| identity.artifact.clone()); - model_ref::format_model_ref(repo, identity.revision.as_deref(), selector.as_deref()) - }) - .or_else(|| { - identity - .canonical_ref - .as_deref() - .and_then(|model_ref| model_ref::ModelRef::parse(model_ref).ok()) - .map(|model_ref| model_ref.display_id()) - }), - ModelSourceKind::Catalog => identity - .canonical_ref - .as_deref() - .and_then(|model_ref| model_ref::ModelRef::parse(model_ref).ok()) - .map(|model_ref| model_ref.display_id()), - ModelSourceKind::LocalGguf | ModelSourceKind::DirectUrl | ModelSourceKind::Unknown => None, - } -} - -pub(crate) fn canonical_demand_model_ref(model: &str) -> String { - if let Ok(model_ref) = model_ref::ModelRef::parse(model) { - return model_ref.display_id(); - } - crate::models::find_loaded_remote_catalog_model_exact(model) - .map(|remote_model| crate::models::remote_catalog_model_ref(&remote_model)) - .unwrap_or_else(|| model.to_string()) -} - /// Peers not directly verified within this window are considered stale /// and excluded from gossip propagation. After 2x this duration they're removed entirely. pub(crate) const PEER_STALE_SECS: u64 = 180; // 3 minutes diff --git a/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs b/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs index 66e8b576ef..9230ab75b8 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs @@ -380,7 +380,8 @@ async fn auto_route_model_has_ready_ingress_target( request_path: &str, affinity: &affinity::AffinityRouter, ) -> bool { - let local_candidates = targets.candidates(model); + let local_candidates = + workload_routing::ingress_candidates(node, model, request_path, targets).await; if contains_routable_candidate(&local_candidates) { return auto_route::model_has_eligible_target( node, @@ -1420,6 +1421,10 @@ mod durable_artifacts; #[path = "ingress_tests/automatic_routing.rs"] mod automatic_routing; +#[cfg(test)] +#[path = "ingress_tests/audio_workloads.rs"] +mod audio_workloads; + #[cfg(test)] #[path = "ingress_tests/tests.rs"] mod tests; diff --git a/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/audio_workloads.rs b/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/audio_workloads.rs new file mode 100644 index 0000000000..81f27685af --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/audio_workloads.rs @@ -0,0 +1,116 @@ +use super::*; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; + +const MODEL: &str = "audio-workload"; + +fn multipart(model: &str) -> Vec { + [b"--test\r\nContent-Disposition: form-data; name=\"file\"; filename=\"audio.wav\"\r\nContent-Type: audio/wav\r\n\r\nRIFF\0{\xff}\x80\r\n--test\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\n".as_slice(), model.as_bytes(), b"\r\n--test--\r\n"].concat() +} + +async fn audio_node() -> mesh::Node { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Worker) + .await + .unwrap(); + node.set_hosted_models(vec![MODEL.into()]).await; + node.set_served_model_descriptors(vec![mesh::ServedModelDescriptor { + identity: mesh::ServedModelIdentity { + model_name: MODEL.into(), + ..Default::default() + }, + capabilities_known: true, + capabilities: crate::models::ModelCapabilities { + audio: crate::models::CapabilityLevel::Supported, + multimodal: true, + ..Default::default() + }, + metadata: Some(mesh::ServedModelMetadata { + workload_class: Some(mesh::ModelWorkloadClass::CausalGeneration), + ..Default::default() + }), + ..Default::default() + }]) + .await; + node +} + +async fn route_audio(path: &str, chunked: bool) { + let backend = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let backend_port = backend.local_addr().unwrap().port(); + let backend_task = tokio::spawn(async move { + let (mut stream, _) = backend.accept().await.unwrap(); + let request = proxy::read_http_request(&mut stream).await.unwrap(); + stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 13\r\nConnection: close\r\n\r\n{\"text\":\"ok\"}").await.unwrap(); + request + }); + let node = audio_node().await; + let mut targets = election::ModelTargets::default(); + targets.targets.insert( + MODEL.into(), + vec![election::InferenceTarget::Local(backend_port)], + ); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let ingress = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + handle_api_proxy_connection( + node, + stream.into(), + targets, + affinity::AffinityRouter::new(), + crate::runtime::IngressType::LocalOpenAi, + ) + .await; + }); + let body = multipart("auto"); + let framing = if chunked { + "Transfer-Encoding: chunked".into() + } else { + format!("Content-Length: {}", body.len()) + }; + let mut raw = format!("POST {path} HTTP/1.1\r\nHost: localhost\r\nContent-Type: multipart/form-data; boundary=test\r\n{framing}\r\n\r\n").into_bytes(); + if chunked { + for chunk in body.chunks(11) { + raw.extend_from_slice(format!("{:x}\r\n", chunk.len()).as_bytes()); + raw.extend_from_slice(chunk); + raw.extend_from_slice(b"\r\n"); + } + raw.extend_from_slice(b"0\r\n\r\n"); + } else { + raw.extend_from_slice(&body); + } + let mut client = TcpStream::connect(address).await.unwrap(); + client.write_all(&raw).await.unwrap(); + let mut response = String::new(); + client.read_to_string(&mut response).await.unwrap(); + assert!(response.starts_with("HTTP/1.1 200"), "{response}"); + let forwarded = backend_task.await.unwrap(); + ingress.await.unwrap(); + assert_eq!(forwarded.model_name.as_deref(), Some(MODEL)); + assert_eq!( + forwarded.body_bytes.as_deref(), + Some(multipart(MODEL).as_slice()) + ); + let header_end = forwarded + .raw + .windows(4) + .position(|bytes| bytes == b"\r\n\r\n") + .unwrap(); + let headers = String::from_utf8_lossy(&forwarded.raw[..header_end]).to_lowercase(); + assert!(!headers.contains("transfer-encoding:")); + assert!(headers.contains(&format!("content-length: {}", multipart(MODEL).len()))); +} + +#[tokio::test] +async fn automatic_audio_reaches_http_backend_with_intact_binary_and_decoded_framing() { + for path in ["/v1/audio/transcriptions", "/v1/audio/translations?trace=1"] { + for chunked in [false, true] { + tokio::time::timeout( + std::time::Duration::from_secs(10), + route_audio(path, chunked), + ) + .await + .expect("audio route must finish"); + } + } +} diff --git a/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/automatic_routing.rs b/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/automatic_routing.rs index 4210bccd28..41de854a02 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/automatic_routing.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/automatic_routing.rs @@ -15,6 +15,35 @@ use super::super::ingress::{ AutoRouteResolution, prepare_cache_routing_body, resolve_auto_routed_model, }; +#[tokio::test] +async fn auto_readiness_uses_remote_embedding_despite_local_causal_copy() { + let model = "shared-workload-model"; + let (node, targets) = node_serving(&[model]).await; + node.set_served_model_descriptors(vec![workload_descriptor( + model, + mesh::ModelWorkloadClass::CausalGeneration, + )]) + .await; + let peer_id = iroh::SecretKey::generate().public(); + let mut peer = peer_serving(peer_id, model, false); + peer.served_model_descriptors = vec![workload_descriptor( + model, + mesh::ModelWorkloadClass::Embedding, + )]; + node.insert_test_peer(peer).await; + assert!( + super::super::ingress::auto_route_model_has_ready_ingress_target( + &node, + &targets, + model, + None, + "/v1/embeddings", + &affinity::AffinityRouter::new() + ) + .await + ); +} + /// A served model with the given capabilities, ready for the media filter. fn descriptor(model: &str, vision: bool, audio: bool) -> mesh::ServedModelDescriptor { use crate::models::CapabilityLevel; diff --git a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/context_selection.rs b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/context_selection.rs index 14b221832a..bccd072556 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/context_selection.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/context_selection.rs @@ -56,6 +56,7 @@ pub(super) async fn eligible_remote_hosts( required_tokens: Option, hosts: Vec, ) -> Vec { + let hosts = super::workload_admission::eligible_remote_hosts(node, model, &hosts).await; let Some(required_tokens) = required_tokens else { return hosts; }; diff --git a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/mod.rs b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/mod.rs index ca9daf337b..d5762b7d79 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/mod.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/mod.rs @@ -60,12 +60,10 @@ async fn degrade_to_single_model( let mut candidates = targets .map(super::ingress::callable_models) .unwrap_or_default(); - if candidates.is_empty() { - candidates = node.models_being_served().await; - } - if candidates.is_empty() { - candidates = node.serving_models().await; - } + candidates.extend(node.models_being_served().await); + candidates.extend(node.serving_models().await); + let descriptors = node.all_served_model_descriptors().await; + candidates.retain(|model| workload_admission::model_supports_committee(model, &descriptors)); let runtimes = node.all_model_runtime_descriptors().await; let Some(target) = context_selection::select_degrade_model(candidates, &runtimes, required_tokens) @@ -258,6 +256,7 @@ mod progress; mod self_fill; mod streaming; mod workers; +mod workload_admission; async fn admitted_gateway_config( node: &mesh::Node, diff --git a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/pool.rs b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/pool.rs index b5843afeb2..a187a84233 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/pool.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/pool.rs @@ -8,6 +8,7 @@ use super::context_selection; use super::self_fill::self_fill_from_extra_instances; use super::workers::{LocalModelBackend, RemoteModelBackend}; +use super::workload_admission; use crate::inference::election; use crate::mesh; use mesh_mixture_of_agents as moa; @@ -36,6 +37,9 @@ enum SizeTier { async fn gossiped_sizes(node: &mesh::Node) -> HashMap { let mut by_base: HashMap = HashMap::new(); for descriptor in node.all_served_model_descriptors().await { + if !workload_admission::descriptor_supports_committee(&descriptor) { + continue; + } if let Some(b) = descriptor .metadata .as_ref() @@ -83,7 +87,11 @@ async fn model_routing_hints( use crate::proto::node::InferenceAdmissionState; let mut hints = HashMap::new(); + let local_descriptors = node.served_model_descriptors().await; for local in node.hosted_models().await { + if !workload_admission::model_supports_committee(&local, &local_descriptors) { + continue; + } hints.insert( canonical_base_name(&local), (AvailabilityRank::Healthy, None), @@ -100,6 +108,10 @@ async fn model_routing_hints( _ => AvailabilityRank::Healthy, }; for model in peer.http_routable_models() { + if !workload_admission::model_supports_committee(&model, &peer.served_model_descriptors) + { + continue; + } let model_base = canonical_base_name(&model); let throughput = peer .advertised_model_throughput @@ -264,7 +276,15 @@ async fn add_worker_backend( }) }) }); - if let Some(port) = local_port { + if let Some(port) = local_port + && !workload_admission::eligible_targets( + resolution.node, + name, + &[election::InferenceTarget::Local(port)], + ) + .await + .is_empty() + { let context_length = resolution.node.local_model_context_length(name).await; if context_selection::context_can_satisfy(resolution.required_tokens, context_length) { let backend_idx = backends.len(); @@ -334,6 +354,7 @@ pub(super) async fn assemble_worker_pool( let mut backends: Vec> = Vec::new(); let mut models: Vec = Vec::new(); let mut local_count = 0usize; + let descriptors = node.all_served_model_descriptors().await; // Full mesh-wide model list (local + every peer's advertised routable // models). @@ -342,6 +363,7 @@ pub(super) async fn assemble_worker_pool( .await .into_iter() .filter(|n| n != moa::VIRTUAL_MODEL_NAME) + .filter(|name| workload_admission::model_supports_committee(name, &descriptors)) .collect(); // Verified sizes gossiped by peers (metadata.parameter_count_b). The @@ -641,6 +663,9 @@ pub(super) async fn compute_actor_candidates( let mut tool_use_by_base: std::collections::HashMap = std::collections::HashMap::new(); for descriptor in node.all_served_model_descriptors().await { + if !workload_admission::descriptor_supports_committee(&descriptor) { + continue; + } let base = canonical_base_name(&descriptor.identity.model_name); let level = descriptor.capabilities.tool_use; tool_use_by_base diff --git a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/self_fill.rs b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/self_fill.rs index f94c01332c..cbfb893c76 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/self_fill.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/self_fill.rs @@ -20,6 +20,8 @@ async fn select_clones( ) -> Vec<(InferenceTarget, Option)> { use crate::proto::node::InferenceAdmissionState; + let candidates = super::workload_admission::eligible_targets(node, name, &candidates).await; + let deprioritized: std::collections::HashSet<_> = node .peers() .await diff --git a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/workload_admission.rs b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/workload_admission.rs new file mode 100644 index 0000000000..fe1005db75 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/workload_admission.rs @@ -0,0 +1,82 @@ +//! Every MoA role sends chat completions, including tool actors and reducers. +//! Encoder-decoder support for standalone generation does not certify that +//! conversational/tool contract. Exclude it, as well as stateless workloads, +//! from every committee role until a separate role capability is defined. +//! Legacy peers without workload metadata retain their historical chat role. + +use crate::inference::election::InferenceTarget; +use crate::mesh::{self, ModelWorkloadClass, ServedModelDescriptor}; + +#[cfg(test)] +mod tests; + +pub(super) fn descriptor_supports_committee(descriptor: &ServedModelDescriptor) -> bool { + matches!( + descriptor + .metadata + .as_ref() + .and_then(|metadata| metadata.workload_class), + None | Some(ModelWorkloadClass::CausalGeneration) + ) +} + +pub(super) fn model_supports_committee(model: &str, descriptors: &[ServedModelDescriptor]) -> bool { + let mut matching = descriptors + .iter() + .filter(|descriptor| { + crate::mesh::model_identity::descriptor_matches_routable_name(descriptor, model) + }) + .peekable(); + matching.peek().is_none() || matching.any(descriptor_supports_committee) +} + +/// Admission is target-local: another peer's descriptor cannot authorize this +/// endpoint. Apply before context ranking, reservations, and standby selection. +pub(super) async fn eligible_targets( + node: &mesh::Node, + model: &str, + candidates: &[InferenceTarget], +) -> Vec { + let local = node.served_model_descriptors().await; + let state = node.state.lock().await; + candidates + .iter() + .filter(|target| { + let descriptors = match target { + InferenceTarget::Local(_) => local.as_slice(), + InferenceTarget::Remote(id) => { + let Some(peer) = state.peers.get(id) else { + return false; + }; + peer.served_model_descriptors.as_slice() + } + InferenceTarget::None => return false, + }; + model_supports_committee(model, descriptors) + }) + .cloned() + .collect() +} + +pub(super) async fn eligible_remote_hosts( + node: &mesh::Node, + model: &str, + hosts: &[iroh::EndpointId], +) -> Vec { + let targets = hosts + .iter() + .copied() + .map(InferenceTarget::Remote) + .collect::>(); + eligible_targets(node, model, &targets) + .await + .into_iter() + .filter_map(|target| { + if let InferenceTarget::Remote(id) = target { + Some(id) + } else { + None + } + }) + .collect() +} diff --git a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/workload_admission/tests.rs b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/workload_admission/tests.rs new file mode 100644 index 0000000000..51c311192a --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/workload_admission/tests.rs @@ -0,0 +1,140 @@ +use super::super::fleet_sim_tests::{BIG_MODELS, SMALL_MODELS, fleet_peer}; +use super::super::pool::{assemble_worker_pool, compute_actor_candidates}; +use super::*; +use crate::inference::election::ModelTargets; +use crate::network::affinity::AffinityRouter; + +fn classified_peer(seed: u32, class: ModelWorkloadClass) -> mesh::PeerInfo { + let mut peer = fleet_peer(seed, BIG_MODELS[0]); + peer.served_model_descriptors[0] + .metadata + .as_mut() + .unwrap() + .workload_class = Some(class); + peer +} + +#[tokio::test] +async fn mixed_workload_fleet_only_admits_chat_models_to_worker_and_actor_roles() { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Client) + .await + .unwrap(); + let classes = [ + ModelWorkloadClass::CausalGeneration, + ModelWorkloadClass::Embedding, + ModelWorkloadClass::Rerank, + ModelWorkloadClass::EncoderDecoder, + ModelWorkloadClass::SpeechSynthesis, + ]; + for (index, class) in classes.into_iter().enumerate() { + let model = if index < 3 { + BIG_MODELS[index] + } else { + SMALL_MODELS[index - 3] + }; + let mut peer = fleet_peer(index as u32 + 1, model); + peer.served_model_descriptors[0] + .metadata + .as_mut() + .unwrap() + .workload_class = Some(class); + node.insert_test_peer(peer).await; + } + let (backends, models) = + assemble_worker_pool(&node, None, None, &reqwest::Client::new(), None).await; + assert_eq!(backends.len(), 1); + assert_eq!(models.len(), 1); + assert_eq!( + super::super::pool::canonical_base_name(&models[0].name), + super::super::pool::canonical_base_name(BIG_MODELS[0].name) + ); + assert_eq!(compute_actor_candidates(&node, &models).await, vec![0]); +} + +#[tokio::test] +async fn target_admission_filters_standbys_and_same_model_clones_before_reservations() { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Worker) + .await + .unwrap(); + let model = BIG_MODELS[0].name; + let local = classified_peer(1, ModelWorkloadClass::EncoderDecoder); + node.set_served_model_descriptors(local.served_model_descriptors) + .await; + let mut targets = ModelTargets::default(); + targets + .targets + .insert(model.into(), vec![InferenceTarget::Local(9337)]); + let mut candidates = vec![InferenceTarget::Local(9337)]; + for (seed, class) in [ + (2, ModelWorkloadClass::Embedding), + (3, ModelWorkloadClass::CausalGeneration), + ] { + let peer = classified_peer(seed, class); + candidates.push(InferenceTarget::Remote(peer.id)); + node.insert_test_peer(peer).await; + } + let eligible = eligible_targets(&node, model, &candidates).await; + assert_eq!(eligible, vec![candidates[2].clone()]); + let remote = match candidates[1] { + InferenceTarget::Remote(id) => id, + _ => unreachable!(), + }; + assert!( + super::super::context_selection::eligible_remote_hosts(&node, model, None, vec![remote]) + .await + .is_empty() + ); + let affinity = AffinityRouter::new(); + let (backends, models) = assemble_worker_pool( + &node, + Some(&targets), + None, + &reqwest::Client::new(), + Some(&affinity), + ) + .await; + assert_eq!( + models.len(), + 1, + "incompatible local/remote copies must not fabricate a committee" + ); + assert_eq!(backends.len(), 1); +} + +#[test] +fn legacy_chat_is_preserved_but_encoder_decoder_never_inherits_a_committee_role() { + assert!(model_supports_committee("legacy", &[])); + let mut descriptor = ServedModelDescriptor::default(); + descriptor.identity.model_name = "model".into(); + assert!(model_supports_committee( + "model", + std::slice::from_ref(&descriptor) + )); + descriptor.metadata = Some(mesh::ServedModelMetadata { + workload_class: Some(ModelWorkloadClass::EncoderDecoder), + ..Default::default() + }); + assert!(!model_supports_committee("model", &[descriptor])); +} + +#[test] +fn public_alias_does_not_bypass_non_chat_admission() { + let mut peer = classified_peer(9, ModelWorkloadClass::Embedding); + let descriptor = &mut peer.served_model_descriptors[0]; + descriptor.identity.source_kind = mesh::ModelSourceKind::HuggingFace; + descriptor.identity.repository = Some("fixture/embedding-GGUF".into()); + descriptor.identity.artifact = Some("embedding.Q8_0.gguf".into()); + let alias = peer.public_model_id_for_routable_model(BIG_MODELS[0].name); + assert_ne!(alias, BIG_MODELS[0].name); + assert!(!model_supports_committee( + &alias, + &peer.served_model_descriptors + )); + assert!( + crate::network::openai::workload_routing::model_satisfies_workload_class( + &alias, + ModelWorkloadClass::Embedding, + &peer.served_model_descriptors + ) + ); +} diff --git a/crates/mesh-llm-host-runtime/src/network/openai/request_parse.rs b/crates/mesh-llm-host-runtime/src/network/openai/request_parse.rs index 84070abc7a..c3ebb98c81 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/request_parse.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/request_parse.rs @@ -11,7 +11,9 @@ use super::request_normalize::{ use super::routing_rank::descriptor_for_model; mod audio_multipart; -use audio_multipart::{multipart_model_field, multipart_model_value_range}; +use audio_multipart::multipart_model_field; +mod body_rewrite; +pub use body_rewrite::{inject_mesh_hooks_flag, rewrite_model_field}; pub(crate) const MAX_HEADER_BYTES: usize = 64 * 1024; /// Private lifecycle ownership assertion used only on trusted mesh forwarding. @@ -990,131 +992,6 @@ pub(super) fn parse_json_body_from_http_request(raw: &[u8]) -> Option, enabled: bool) { - let Some(header_end) = raw.windows(4).position(|w| w == b"\r\n\r\n").map(|i| i + 4) else { - return; - }; - let body = &raw[header_end..]; - let Some(brace) = body.iter().position(|&b| b == b'{') else { - return; - }; - - // Build new body with mesh_hooks injected after opening brace - let fragment = if enabled { - &b"\"mesh_hooks\":true,"[..] - } else { - &b"\"mesh_hooks\":false,"[..] - }; - let mut new_body = Vec::with_capacity(body.len() + fragment.len()); - new_body.extend_from_slice(&body[..brace + 1]); - new_body.extend_from_slice(fragment); - new_body.extend_from_slice(&body[brace + 1..]); - - // Rebuild headers with correct Content-Length - let headers = std::str::from_utf8(&raw[..header_end - 4]).unwrap_or(""); - let mut rebuilt = String::new(); - for line in headers.split("\r\n") { - if line.to_ascii_lowercase().starts_with("content-length:") { - rebuilt.push_str(&format!("Content-Length: {}", new_body.len())); - } else { - rebuilt.push_str(line); - } - rebuilt.push_str("\r\n"); - } - rebuilt.push_str("\r\n"); - - let mut result = rebuilt.into_bytes(); - result.extend_from_slice(&new_body); - *raw = result; -} - -fn content_type_from_request(raw: &[u8]) -> Option { - let mut headers = [httparse::EMPTY_HEADER; MAX_HEADERS]; - let mut parsed = httparse::Request::new(&mut headers); - let httparse::Status::Complete(_) = parsed.parse(raw).ok()? else { - return None; - }; - parsed - .headers - .iter() - .find(|header| header.name.eq_ignore_ascii_case("content-type")) - .and_then(|header| std::str::from_utf8(header.value).ok()) - .map(str::to_string) -} - -fn rebuild_request_body( - request: &mut BufferedHttpRequest, - header_end: usize, - new_body: Vec, - body_json: Option, - model: &str, -) { - let headers = std::str::from_utf8(&request.raw[..header_end - 4]).unwrap_or(""); - let mut rebuilt = String::new(); - for line in headers.split("\r\n") { - if line.to_ascii_lowercase().starts_with("content-length:") { - rebuilt.push_str(&format!("Content-Length: {}", new_body.len())); - } else { - rebuilt.push_str(line); - } - rebuilt.push_str("\r\n"); - } - rebuilt.push_str("\r\n"); - - let mut raw = rebuilt.into_bytes(); - raw.extend_from_slice(&new_body); - request.raw = raw; - request.body_len_bytes = new_body.len(); - request.body_bytes = Some(new_body); - request.body_json = body_json; - request.body_json_attempted = true; - request.model_name = Some(model.to_string()); -} - -/// Rewrite the JSON or multipart `model` field and rebuild Content-Length. -pub fn rewrite_model_field(request: &mut BufferedHttpRequest, model: &str) { - let Some(header_end) = request - .raw - .windows(4) - .position(|w| w == b"\r\n\r\n") - .map(|i| i + 4) - else { - return; - }; - - if let Ok(mut body) = serde_json::from_slice::(&request.raw[header_end..]) { - let Some(object) = body.as_object_mut() else { - return; - }; - object.insert( - "model".to_string(), - serde_json::Value::String(model.to_string()), - ); - let Ok(new_body) = serde_json::to_vec(&body) else { - return; - }; - rebuild_request_body(request, header_end, new_body, Some(body), model); - return; - } - - let Some(content_type) = content_type_from_request(&request.raw) else { - return; - }; - let original = &request.raw[header_end..]; - let Ok(Some(range)) = multipart_model_value_range(&content_type, original) else { - return; - }; - let mut new_body = Vec::with_capacity(original.len() - range.len() + model.len()); - new_body.extend_from_slice(&original[..range.start]); - new_body.extend_from_slice(model.as_bytes()); - new_body.extend_from_slice(&original[range.end..]); - rebuild_request_body(request, header_end, new_body, None, model); -} - pub fn is_models_list_request(method: &str, path: &str) -> bool { let path = path.split('?').next().unwrap_or(path); method == "GET" && (path == "/v1/models" || path == "/models") diff --git a/crates/mesh-llm-host-runtime/src/network/openai/request_parse/body_rewrite.rs b/crates/mesh-llm-host-runtime/src/network/openai/request_parse/body_rewrite.rs new file mode 100644 index 0000000000..f56b8e313b --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/network/openai/request_parse/body_rewrite.rs @@ -0,0 +1,159 @@ +//! Body transformations operate on decoded payloads, never HTTP chunk framing. + +use super::audio_multipart::multipart_model_value_range; +use super::{BufferedHttpRequest, MAX_BODY_BYTES, MAX_HEADERS, try_decode_chunked_body}; + +#[cfg(test)] +mod tests; + +struct BodyHeaders<'a> { + end: usize, + content_type: Option<&'a str>, + chunked: bool, +} + +impl BodyHeaders<'_> { + fn permits_json(&self) -> bool { + self.content_type.is_none_or(|value| { + value.split(';').next().is_some_and(|media_type| { + media_type.trim().eq_ignore_ascii_case("application/json") + }) + }) + } +} + +fn body_headers(raw: &[u8]) -> Option> { + let mut headers = [httparse::EMPTY_HEADER; MAX_HEADERS]; + let mut parsed = httparse::Request::new(&mut headers); + let httparse::Status::Complete(end) = parsed.parse(raw).ok()? else { + return None; + }; + let content_type = parsed + .headers + .iter() + .find(|header| header.name.eq_ignore_ascii_case("content-type")) + .map(|header| std::str::from_utf8(header.value)) + .transpose() + .ok()?; + let chunked = parsed + .headers + .iter() + .any(|header| header.name.eq_ignore_ascii_case("transfer-encoding")); + Some(BodyHeaders { + end, + content_type, + chunked, + }) +} + +/// Replace framing with one Content-Length, preserving all other header bytes. +fn replace_body(raw: &mut Vec, header_end: usize, body: &[u8]) { + let mut rebuilt = Vec::with_capacity(header_end + body.len()); + for line in raw[..header_end].split_inclusive(|byte| *byte == b'\n') { + let name = line.split(|byte| *byte == b':').next().unwrap_or_default(); + if line == b"\r\n" + || line == b"\n" + || name.eq_ignore_ascii_case(b"content-length") + || name.eq_ignore_ascii_case(b"transfer-encoding") + || name.eq_ignore_ascii_case(b"trailer") + { + continue; + } + rebuilt.extend_from_slice(line); + } + rebuilt.extend_from_slice(format!("Content-Length: {}\r\n\r\n", body.len()).as_bytes()); + rebuilt.extend_from_slice(body); + *raw = rebuilt; +} + +/// Set the hook flag only on an actual JSON object. Multipart and binary +/// requests remain byte-identical, even when their media contains `{`. +pub fn inject_mesh_hooks_flag(raw: &mut Vec, enabled: bool) { + let Some(headers) = body_headers(raw) else { + return; + }; + if !headers.permits_json() { + return; + } + let decoded; + let body = if headers.chunked { + let Ok(Some((_, bytes))) = try_decode_chunked_body(&raw[headers.end..], MAX_BODY_BYTES) + else { + return; + }; + decoded = bytes; + &decoded[..] + } else { + &raw[headers.end..] + }; + let Ok(mut json) = serde_json::from_slice::(body) else { + return; + }; + let Some(object) = json.as_object_mut() else { + return; + }; + object.insert("mesh_hooks".into(), enabled.into()); + let Ok(body) = serde_json::to_vec(&json) else { + return; + }; + let end = headers.end; + replace_body(raw, end, &body); +} + +fn rebuild_request_body( + request: &mut BufferedHttpRequest, + header_end: usize, + body: Vec, + json: Option, + model: &str, +) { + replace_body(&mut request.raw, header_end, &body); + request.body_len_bytes = body.len(); + request.body_bytes = Some(body); + request.body_json = json; + request.body_json_attempted = true; + request.model_name = Some(model.to_string()); +} + +/// Rewrite the JSON or multipart model field using the reader's decoded body +/// for chunked uploads. Only the model part changes; binary media stays intact. +pub fn rewrite_model_field(request: &mut BufferedHttpRequest, model: &str) { + let Some(headers) = body_headers(&request.raw) else { + return; + }; + let original = if headers.chunked { + let Some(body) = request.body_bytes.as_deref() else { + return; + }; + body + } else { + &request.raw[headers.end..] + }; + if headers.permits_json() { + let Ok(mut json) = serde_json::from_slice::(original) else { + return; + }; + let Some(object) = json.as_object_mut() else { + return; + }; + object.insert("model".into(), model.into()); + let Ok(body) = serde_json::to_vec(&json) else { + return; + }; + let end = headers.end; + rebuild_request_body(request, end, body, Some(json), model); + return; + } + let Some(content_type) = headers.content_type else { + return; + }; + let Ok(Some(range)) = multipart_model_value_range(content_type, original) else { + return; + }; + let mut body = Vec::with_capacity(original.len() - range.len() + model.len()); + body.extend_from_slice(&original[..range.start]); + body.extend_from_slice(model.as_bytes()); + body.extend_from_slice(&original[range.end..]); + let end = headers.end; + rebuild_request_body(request, end, body, None, model); +} diff --git a/crates/mesh-llm-host-runtime/src/network/openai/request_parse/body_rewrite/tests.rs b/crates/mesh-llm-host-runtime/src/network/openai/request_parse/body_rewrite/tests.rs new file mode 100644 index 0000000000..89abdd38f4 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/network/openai/request_parse/body_rewrite/tests.rs @@ -0,0 +1,117 @@ +use super::*; +use tokio::io::AsyncWriteExt; + +async fn read_request(raw: Vec) -> BufferedHttpRequest { + let (mut writer, mut reader) = tokio::io::duplex(4096); + let write = tokio::spawn(async move { + writer.write_all(&raw).await.unwrap(); + }); + let request = crate::network::openai::request_parse::read_http_request(&mut reader) + .await + .expect("valid request"); + write.await.unwrap(); + request +} + +fn multipart(model: &str) -> Vec { + [ + b"--audio\r\nContent-Disposition: form-data; name=\"file\"; filename=\"audio.wav\"\r\nContent-Type: audio/wav\r\n\r\nRIFF\0{\xff}\x80\r\n--audio\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\n".as_slice(), + model.as_bytes(), b"\r\n--audio--\r\n", + ].concat() +} + +fn raw_request(path: &str, content_type: &str, body: &[u8], chunked: bool) -> Vec { + let framing = if chunked { + "Transfer-Encoding: chunked\r\nTrailer: X-Checksum".into() + } else { + format!("Content-Length: {}", body.len()) + }; + let mut raw = format!("POST {path} HTTP/1.1\r\nHost: localhost\r\nContent-Type: {content_type}\r\n{framing}\r\n\r\n").into_bytes(); + if chunked { + for chunk in body.chunks(7) { + raw.extend_from_slice(format!("{:x};test=1\r\n", chunk.len()).as_bytes()); + raw.extend_from_slice(chunk); + raw.extend_from_slice(b"\r\n"); + } + raw.extend_from_slice(b"0\r\nX-Checksum: fixture\r\n\r\n"); + } else { + raw.extend_from_slice(body); + } + raw +} + +#[tokio::test] +async fn automatic_audio_rewrite_preserves_binary_bytes_and_replaces_chunk_framing() { + for path in ["/v1/audio/transcriptions", "/v1/audio/translations?trace=1"] { + for chunked in [false, true] { + let raw = raw_request( + path, + "multipart/form-data; boundary=audio", + &multipart("auto"), + chunked, + ); + let mut request = read_request(raw).await; + assert_eq!(request.model_name.as_deref(), Some("auto")); + // Host ingress injects first; the passive path injects after rewriting. + let original = request.raw.clone(); + inject_mesh_hooks_flag(&mut request.raw, true); + assert_eq!( + request.raw, original, + "hooks must not modify audio or framing" + ); + rewrite_model_field(&mut request, "audio-model"); + inject_mesh_hooks_flag(&mut request.raw, true); + let headers = body_headers(&request.raw).unwrap(); + assert!(!headers.chunked); + let body = &request.raw[headers.end..]; + assert_eq!(body, multipart("audio-model")); + assert_eq!(request.body_bytes.as_deref(), Some(body)); + assert_eq!(request.body_len_bytes, body.len()); + assert!(request.body_json.is_none()); + let header_text = std::str::from_utf8(&request.raw[..headers.end]) + .unwrap() + .to_lowercase(); + assert_eq!(header_text.matches("content-length:").count(), 1); + assert!(header_text.contains(&format!("content-length: {}\r\n", body.len()))); + assert!(!header_text.contains("trailer:")); + } + } +} + +#[test] +fn hook_injection_rejects_non_json_and_sets_one_valid_flag() { + for (content_type, body) in [ + ("application/octet-stream", b"{\"binary\":true}".as_slice()), + ("application/json", b"prefix{\"invalid\":true}".as_slice()), + ("application/json", b"[{}]".as_slice()), + ] { + let mut raw = raw_request("/v1/chat/completions", content_type, body, false); + let before = raw.clone(); + inject_mesh_hooks_flag(&mut raw, true); + assert_eq!(raw, before); + } + for body in [ + b"{}".as_slice(), + b"{\"mesh_hooks\":false,\"model\":\"auto\"}".as_slice(), + ] { + for chunked in [false, true] { + let mut raw = raw_request( + "/v1/chat/completions", + "application/json; charset=utf-8", + body, + chunked, + ); + inject_mesh_hooks_flag(&mut raw, true); + let headers = body_headers(&raw).unwrap(); + assert!(!headers.chunked); + let json: serde_json::Value = serde_json::from_slice(&raw[headers.end..]).unwrap(); + assert_eq!(json["mesh_hooks"], true); + assert_eq!( + String::from_utf8_lossy(&raw[headers.end..]) + .matches("mesh_hooks") + .count(), + 1 + ); + } + } +} diff --git a/crates/mesh-llm-host-runtime/src/network/openai/request_parse_tests.rs b/crates/mesh-llm-host-runtime/src/network/openai/request_parse_tests.rs index d04120c993..f94ca43e7d 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/request_parse_tests.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/request_parse_tests.rs @@ -869,7 +869,7 @@ async fn multipart_model_is_parsed_and_rewritten_without_touching_file_bytes() { .position(|window| window == b"\r\n\r\n") .unwrap() + 4; - let content_type = content_type_from_request(&request.raw).unwrap(); + let content_type = format!("multipart/form-data; boundary={BOUNDARY}"); assert_eq!( multipart_model_field(&content_type, &request.raw[header_end..]) .unwrap() diff --git a/crates/mesh-llm-host-runtime/src/network/openai/transport.rs b/crates/mesh-llm-host-runtime/src/network/openai/transport.rs index b978a9f89a..cfcb58703f 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/transport.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/transport.rs @@ -597,7 +597,7 @@ fn prepare_mesh_targets( if !request.is_tokenize_request() && effective_model.is_some() && !target_hosts.is_empty() { request.ensure_body_json(); } - let body_json = request.body_json.as_ref(); + let body_json = workload_routing::affinity_body(request); effective_model .map(|name| prepare_remote_targets_for_request(name, target_hosts, body_json, affinity)) .unwrap_or(PreparedTargets { @@ -1161,7 +1161,7 @@ fn auto_session_key_for_request( request: &mut BufferedHttpRequest, is_auto_request: bool, ) -> Option { - if !is_auto_request { + if !is_auto_request || !workload_routing::supports_generation_affinity(&request.client_path) { return None; } request.ensure_body_json(); diff --git a/crates/mesh-llm-host-runtime/src/network/openai/transport_route_model.rs b/crates/mesh-llm-host-runtime/src/network/openai/transport_route_model.rs index 635714bc04..c8a96b5d22 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/transport_route_model.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/transport_route_model.rs @@ -95,11 +95,11 @@ async fn route_model_request_inner(args: RouteModelRequestArgs<'_>) -> RouteDisp } = args; let route_started = Instant::now(); let mut tcp_stream = tcp_stream; - let candidates = super::super::workload_routing::eligible_targets( + let candidates = super::super::workload_routing::ingress_candidates( &node, model, &request.client_path, - &targets.candidates(model), + targets, ) .await; let ranked = rank_targets_by_context(&node, model, required_tokens, &candidates).await; @@ -114,7 +114,8 @@ async fn route_model_request_inner(args: RouteModelRequestArgs<'_>) -> RouteDisp } route_observer.route_selected(Some(model)); - let prefix_hash = crate::network::affinity::cache_prefix_hash(request.body_json.as_ref()); + let affinity_body = super::super::workload_routing::affinity_body(request); + let prefix_hash = crate::network::affinity::cache_prefix_hash(affinity_body); let cache_target = cache_target_for_request(&node, affinity, model, prefix_hash, &ordered_candidates).await; let Some(ReservedModelRoute { @@ -125,7 +126,7 @@ async fn route_model_request_inner(args: RouteModelRequestArgs<'_>) -> RouteDisp targets, &ranked, model, - request.body_json.as_ref(), + affinity_body, affinity, cache_target, ) diff --git a/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/workload_routing.rs b/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/workload_routing.rs index 58a6486c59..86ff1b7a36 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/workload_routing.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/workload_routing.rs @@ -7,6 +7,54 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; const MODEL: &str = "shared-workload-model"; +#[tokio::test] +async fn explicit_ingress_falls_back_to_capable_remote_when_local_workload_is_incompatible() { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Worker) + .await + .expect("node"); + let remote = peer(&node, Some(ModelWorkloadClass::Embedding), false).await; + for class in [None, Some(ModelWorkloadClass::CausalGeneration)] { + node.set_served_model_descriptors(vec![descriptor(class, false)]) + .await; + let mut targets = election::ModelTargets::default(); + targets + .targets + .insert(MODEL.into(), vec![election::InferenceTarget::Local(9337)]); + let selected = + workload_routing::ingress_candidates(&node, MODEL, "/v1/embeddings", &targets).await; + assert_eq!(selected, vec![election::InferenceTarget::Remote(remote)]); + } +} + +#[tokio::test] +async fn stateless_user_metadata_does_not_disable_replica_reservation_spreading() { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Client) + .await + .expect("node"); + for (path, class) in [ + ("/v1/embeddings", ModelWorkloadClass::Embedding), + ("/v1/rerank", ModelWorkloadClass::Rerank), + ("/v1/audio/speech", ModelWorkloadClass::SpeechSynthesis), + ] { + let first = peer(&node, Some(class), false).await; + let second = peer(&node, Some(class), false).await; + let affinity = AffinityRouter::new(); + let mut request = request(path, MODEL); + let plan = build_mesh_request_plan(&node, &mut request, false, &affinity) + .await + .unwrap_or_else(|_| panic!("compatible replicas")); + assert!(!plan.affinity_applied); + assert_eq!(plan.equivalent_hosts, 2); + assert!(plan.target_hosts.contains(&first) && plan.target_hosts.contains(&second)); + let (first_hosts, _first_reservation) = reserve_mesh_request_target(&plan, &affinity); + let (second_hosts, _second_reservation) = reserve_mesh_request_target(&plan, &affinity); + assert_ne!( + first_hosts[0], second_hosts[0], + "concurrent stateless requests must spread" + ); + } +} + fn descriptor(class: Option, audio: bool) -> mesh::ServedModelDescriptor { mesh::ServedModelDescriptor { identity: mesh::ServedModelIdentity { @@ -241,7 +289,9 @@ async fn passive_auto_model_cache_cannot_cross_workload_boundaries() { .await; let affinity = AffinityRouter::new(); let mut request = request("/v1/embeddings", "auto"); - let key = auto_session_key_for_request(&mut request, true).expect("explicit cache key"); + let key = crate::network::affinity::auto_model_session_key(request.body_json.as_ref()) + .expect("legacy cache key"); + assert_eq!(auto_session_key_for_request(&mut request, true), None); affinity.remember_auto_model(key, "legacy-chat"); // No healthy compatible alternative: the availability fallback must still // stay inside the requested workload, never restore the cached chat model. @@ -255,7 +305,10 @@ async fn passive_auto_model_cache_cannot_cross_workload_boundaries() { .unwrap_or_else(|_| panic!("embedding fallback remains available")); assert_eq!(plan.effective_model.as_deref(), Some(MODEL)); assert_eq!(plan.target_hosts, vec![capable]); - assert_eq!(affinity.lookup_auto_model(key).as_deref(), Some(MODEL)); + assert_eq!( + affinity.lookup_auto_model(key).as_deref(), + Some("legacy-chat") + ); } #[tokio::test] @@ -279,11 +332,10 @@ async fn audio_upload_capabilities_must_belong_to_one_descriptor_on_the_target() } #[tokio::test] -async fn host_dispatch_rejects_local_legacy_target_despite_capable_remote_metadata() { +async fn host_dispatch_rejects_local_legacy_target_without_capable_replicas() { let node = mesh::Node::new_for_tests(mesh::NodeRole::Worker) .await .expect("node"); - peer(&node, Some(ModelWorkloadClass::Embedding), false).await; node.set_served_model_descriptors(vec![descriptor(None, false)]) .await; let backend = tokio::net::TcpListener::bind("127.0.0.1:0") diff --git a/crates/mesh-llm-host-runtime/src/network/openai/workload_routing.rs b/crates/mesh-llm-host-runtime/src/network/openai/workload_routing.rs index 29f482bf48..1b3e70be55 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/workload_routing.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/workload_routing.rs @@ -6,6 +6,7 @@ //! context ranking, affinity, reservations, and retry selection. use crate::inference::election::InferenceTarget; +use crate::mesh::model_identity::descriptor_matches_routable_name; use crate::mesh::{self, ModelWorkloadClass, ServedModelDescriptor}; #[cfg(test)] @@ -32,6 +33,47 @@ pub(super) fn request_workload_class(path: &str) -> Option { } } +/// Only generation requests reuse KV/session state. Metadata such as an +/// embeddings `user` field must not pin a stateless workload to one replica. +pub(super) fn supports_generation_affinity(path: &str) -> bool { + matches!( + path.split('?').next().unwrap_or(path), + "/v1/chat/completions" | "/v1/completions" | "/v1/responses" + ) +} + +pub(super) fn affinity_body( + request: &super::request_parse::BufferedHttpRequest, +) -> Option<&serde_json::Value> { + supports_generation_affinity(&request.client_path) + .then_some(request.body_json.as_ref()) + .flatten() +} + +/// Prefer the elected targets only when at least one supports this request. +/// A stale/local incompatible copy must not shadow capable remote replicas. +pub(super) async fn ingress_candidates( + node: &mesh::Node, + model: &str, + path: &str, + targets: &crate::inference::election::ModelTargets, +) -> Vec { + let local = eligible_targets(node, model, path, &targets.candidates(model)).await; + if local + .iter() + .any(|target| !matches!(target, InferenceTarget::None)) + { + return local; + } + let remote = node + .hosts_for_model(model) + .await + .into_iter() + .map(InferenceTarget::Remote) + .collect::>(); + eligible_targets(node, model, path, &remote).await +} + fn class_is_compatible( requested: ModelWorkloadClass, advertised: Option, @@ -54,7 +96,7 @@ pub(super) fn model_satisfies_workload_class( ) -> bool { let mut matching = descriptors .iter() - .filter(|descriptor| descriptor.identity.model_name == model) + .filter(|descriptor| descriptor_matches_routable_name(descriptor, model)) .peekable(); if matching.peek().is_none() { return class_is_compatible(requested, None); @@ -90,7 +132,8 @@ pub(super) fn model_satisfies_request_workload( ) -> bool { if is_audio_upload_path(path) { descriptors.iter().any(|descriptor| { - descriptor.identity.model_name == model && descriptor_supports_audio_upload(descriptor) + descriptor_matches_routable_name(descriptor, model) + && descriptor_supports_audio_upload(descriptor) }) } else { model_satisfies_workload_class(model, workload, descriptors) @@ -103,7 +146,7 @@ pub(super) fn descriptor_for_request<'a>( descriptors: &'a [ServedModelDescriptor], ) -> Option<&'a ServedModelDescriptor> { descriptors.iter().find(|descriptor| { - descriptor.identity.model_name == model + descriptor_matches_routable_name(descriptor, model) && request_workload_class(path).is_none_or(|workload| { model_satisfies_request_workload( model, From d97715e2a90c6ecefc1c349e03be3a285fb6385a Mon Sep 17 00:00:00 2001 From: IvGolovach <20299097+IvGolovach@users.noreply.github.com> Date: Sat, 12 Sep 2026 16:15:58 -0700 Subject: [PATCH 05/18] ci(skippy): provision source-bound CPU workload canaries --- .../manage-ci/references/current-inventory.md | 9 +- .github/workflows/llama-upstream-canary.yml | 9 ++ ci/ci.md | 14 ++- ci/llama-canary/family-certified.json | 12 +-- .../manifests/competitive-benchmark.json | 2 +- .../manifests/hf-download-smoke.json | 2 +- .../manifests/openai-smoke.json | 2 +- .../manifests/product-integration-smoke.json | 2 +- .../manifests/product-smoke.json | 2 +- ci/model-artifacts/manifests/radix-cache.json | 2 +- .../manifests/safetensors-runtime-smoke.json | 2 +- .../manifests/scripted-binary-smoke.json | 2 +- ci/model-artifacts/manifests/sdk-smoke.json | 2 +- .../manifests/skippy-ci-smoke.json | 2 +- .../manifests/skippy-correctness.json | 2 +- .../manifests/skippy-parity.json | 2 +- ci/model-artifacts/registry.json | 12 +++ just/skippy.just | 5 ++ scripts/check-skippy-workload-candidate.py | 87 ++++++++++++++++++- scripts/llama-canary-agent-repair.sh | 8 ++ scripts/skippy-family-battery.sh | 5 +- scripts/skippy-workload-certify.sh | 43 ++++++--- scripts/skippy-workload-oracles-build.sh | 49 +++++++++++ .../test_check_skippy_workload_candidate.py | 41 +++++++++ scripts/tests/test_justfile_layout.py | 1 + scripts/tests/test_plan_family_battery.py | 2 +- .../test_skippy_workload_oracles_build.py | 52 +++++++++++ 27 files changed, 336 insertions(+), 37 deletions(-) create mode 100644 scripts/skippy-workload-oracles-build.sh create mode 100644 scripts/tests/test_skippy_workload_oracles_build.py diff --git a/.agents/skills/manage-ci/references/current-inventory.md b/.agents/skills/manage-ci/references/current-inventory.md index d67a04996e..44af0c4124 100644 --- a/.agents/skills/manage-ci/references/current-inventory.md +++ b/.agents/skills/manage-ci/references/current-inventory.md @@ -80,7 +80,14 @@ that every binary already exists. A scheduled unchanged pin selects the four `nightly` cache-mechanism sentinels (Qwen3 dense, Falcon-H1, Qwen3Next, and Mamba). A changed pin selects `llama-bump`; a manual dispatch may set `force_certify` to select `manual-full`. Both latter cadences retain the full -family battery. Before any certification starts, every selected GGUF is resolved +family battery, including all six `workload-oracle` rows. Both normal and +independent changed-pin verification explicitly build a run-scoped CPU oracle +closure with `just skippy-workload-oracles-build`: monolithic server/completion/ +TTS references plus separate static CPU candidate and test binaries. Generated +`SKIPPY_WORKLOAD_*` paths select that closure without replacing Metal outputs. +The source- and executable-bound `producer.json` is checked before consuming +prebuilt workload binaries; `--skip-build` never silently rebuilds them. +Before any certification starts, every selected GGUF is resolved directly by the immutable snapshot SHA checked into `ci/llama-canary/family-certified.json`. The runtime preflight records the revisions and verifies all shard/tensor scans, declared runtime/MTP layer diff --git a/.github/workflows/llama-upstream-canary.yml b/.github/workflows/llama-upstream-canary.yml index 24a3f7fdfc..ddf6956984 100644 --- a/.github/workflows/llama-upstream-canary.yml +++ b/.github/workflows/llama-upstream-canary.yml @@ -317,6 +317,14 @@ jobs: SKIPPY_CANARY_LIVE_MATRIX_ROOT: ${{ github.workspace }}/target/family-battery/${{ github.run_id }}-${{ github.run_attempt }} run: scripts/skippy-canary-live-matrix.sh --prepare + - name: Build pinned CPU workload oracles and candidate + if: steps.prepare.outcome == 'success' && steps.sha.outputs.certify == 'true' && steps.family_patch.outcome == 'success' && steps.sha.outputs.cadence != 'nightly' + run: | + set -euo pipefail + workload_root="${LLAMA_STAGE_BUILD_DIR:?}-workloads" + arch -arm64 just skippy-workload-oracles-build "$workload_root" + bash scripts/skippy-workload-oracles-build.sh --print-env "$workload_root" >> "$GITHUB_ENV" + - name: Supported-families certification battery (parity gate) id: battery if: steps.prepare.outcome == 'success' && steps.sha.outputs.certify == 'true' && steps.family_patch.outcome == 'success' @@ -338,6 +346,7 @@ jobs: name: llama-family-battery-${{ github.run_id }}-${{ github.run_attempt }} path: | target/family-battery/${{ github.run_id }}-${{ github.run_attempt }}/ + ${{ env.LLAMA_STAGE_BUILD_DIR }}-workloads/producer.json target/skippy-stage-rewriter-check/ if-no-files-found: warn retention-days: 14 diff --git a/ci/ci.md b/ci/ci.md index 4b1992ee3f..937edec111 100644 --- a/ci/ci.md +++ b/ci/ci.md @@ -60,7 +60,19 @@ Scheduled coverage details: an unchanged-pin llama canary uses the bounded `nightly` cadence (Qwen3 dense, Falcon-H1, Qwen3Next, and Mamba). Changed pins use the complete `llama-bump` cohort, and a forced dispatch of the unchanged pin uses `manual-full`. Both latter paths retain the complete supported-family -certification described in the table. The +certification described in the table, including all six non-chat +`workload-oracle` rows. Both ordinary and independently verified changed-pin +canaries run `just skippy-workload-oracles-build` in a run-specific directory. +It produces pinned CPU `llama-server`, `llama-completion`, and `llama-tts` +references plus a separate static CPU `skippy-server` and test binary, without +overwriting the Metal family outputs. Generated `SKIPPY_WORKLOAD_*` paths +replace ambient runner configuration. `producer.json` binds the executables +and native stamp to the repository head/worktree; consumers verify it and do +not rebuild under `--skip-build`. Model/projector cache integrity is still +checked before execution, and populating the read-only lab cache remains an +external runner operation. + +The competitive benchmark can optionally download exact-cohort history from `MESH_PERFORMANCE_HISTORY_DATASET`, validate the checked-in schema, report regression candidates, and append one immutable run shard using diff --git a/ci/llama-canary/family-certified.json b/ci/llama-canary/family-certified.json index a12211aa31..54ff3b19d7 100644 --- a/ci/llama-canary/family-certified.json +++ b/ci/llama-canary/family-certified.json @@ -810,7 +810,7 @@ "family": "nomic-bert-embedding", "class": "embedding", "profile": "workload-oracle", - "cadences": ["manual-full"], + "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "nomic-ai/nomic-embed-text-v1.5-GGUF", "revision": "0188c9bf409793f810680a5a431e7b899c46104c", "files": ["nomic-embed-text-v1.5.Q8_0.gguf"], "file_integrity": {"nomic-embed-text-v1.5.Q8_0.gguf": {"size_bytes": 146146432, "blob_id": "3e24342164b3d94991ba9692fdc0dd08e3fd7362e0aacc396a9a5c54a544c3b7"}}, "selector": "Q8_0"}, "evidence": {"fixture": "scripts/workload_fixtures.py#EMBEDDING_INPUTS", "comparison": "batch and individual vectors within 1e-4 coordinate error and 0.99999 cosine"}, "execution": {"trunk_layers": 12, "mtp_layers": 0, "activation_width": 768, "boundary_sweep_period": 0, "speculative_policy": "disabled"}, @@ -821,7 +821,7 @@ "family": "jina-bert-v2-rerank", "class": "rerank", "profile": "workload-oracle", - "cadences": ["manual-full"], + "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "ggml-org/jina-reranker-v1-turbo-en-GGUF", "revision": "607d8664c787e517e5d6e339d21f680f9002c931", "files": ["Jina-Bert-Implementation-38M-F16.gguf"], "file_integrity": {"Jina-Bert-Implementation-38M-F16.gguf": {"size_bytes": 76971168, "blob_id": "71abc010bb3dce97812ee971509a5cb6ff6f6b8cfffd8480129242f605521fca"}}, "selector": "F16"}, "evidence": {"fixture": "scripts/workload_fixtures.py#RERANK_DOCUMENTS", "comparison": "scores within 1e-4 and identical document order"}, "execution": {"trunk_layers": 6, "mtp_layers": 0, "activation_width": 384, "boundary_sweep_period": 0, "speculative_policy": "disabled"}, @@ -832,7 +832,7 @@ "family": "t5-encoder-decoder", "class": "encoder_decoder", "profile": "workload-oracle", - "cadences": ["manual-full"], + "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "Felladrin/gguf-flan-t5-small", "revision": "d71c51f67519edd3154527c2d8f20288bdde9705", "files": ["flan-t5-small.Q8_0.gguf"], "file_integrity": {"flan-t5-small.Q8_0.gguf": {"size_bytes": 113709824, "blob_id": "f7f769c360b1ba830b10dd3b7e7d146dbcc4d487962be7dd806d7d52e0a9c2f0"}}, "selector": "Q8_0"}, "evidence": {"fixture": "scripts/workload_fixtures.py#ENCODER_DECODER_PROMPT", "comparison": "identical normalized greedy text from pinned llama-completion"}, "execution": {"trunk_layers": 8, "mtp_layers": 0, "activation_width": 512, "boundary_sweep_period": 0, "speculative_policy": "disabled"}, @@ -843,7 +843,7 @@ "family": "paddleocr", "class": "ocr", "profile": "workload-oracle", - "cadences": ["manual-full"], + "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "PaddlePaddle/PaddleOCR-VL-1.6-GGUF", "revision": "511b09642bb324401f15f97cc23bc67e8f0a291d", "files": ["PaddleOCR-VL-1.6-GGUF.gguf"], "file_integrity": {"PaddleOCR-VL-1.6-GGUF.gguf": {"size_bytes": 935769056, "blob_id": "f3ae46ec885050acf4b3d31944431e1fd90d50664fb09126af4a3c050ba14ee8"}}, "selector": "BF16"}, "mmproj_artifact": {"repo": "PaddlePaddle/PaddleOCR-VL-1.6-GGUF", "revision": "511b09642bb324401f15f97cc23bc67e8f0a291d", "files": ["PaddleOCR-VL-1.6-GGUF-mmproj.gguf"], "file_integrity": {"PaddleOCR-VL-1.6-GGUF-mmproj.gguf": {"size_bytes": 881770560, "blob_id": "204d757d7610d9b3faab10d506d69e5b244e32bf765e2bab2d0167e65e0a058a"}}, "selector": "BF16"}, "evidence": {"fixture": "scripts/generate-ocr-oracle-fixture.py#MESH 42", "comparison": "normalized monolithic text parity plus independent MESH 42 label"}, @@ -855,7 +855,7 @@ "family": "qwen3tts", "class": "speech_synthesis", "profile": "workload-oracle", - "cadences": ["manual-full"], + "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF", "revision": "ca27d74bc954b73dadab5b71ca265d87fc861a7c", "files": ["Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf"], "file_integrity": {"Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf": {"size_bytes": 1847874400, "blob_id": "ac7931aeb2e7aad1a6ed6602d353a5679c9d096b18ce8204ac730a8408d572e1"}}, "selector": "Q8_0"}, "mmproj_artifact": {"repo": "ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF", "revision": "ca27d74bc954b73dadab5b71ca265d87fc861a7c", "files": ["mmproj-Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf"], "file_integrity": {"mmproj-Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf": {"size_bytes": 446422912, "blob_id": "6fd65188839bcd6ecc91b277ad471e22a0edfada4699a0fe82f1165c18cfcce2"}}, "selector": "Q8_0"}, "evidence": {"fixture": "scripts/skippy-tts-oracle.py#PROMPT", "comparison": "fixed-seed PCM format and length; RMS error at most 2 percent and waveform cosine at least 0.9995"}, @@ -867,7 +867,7 @@ "family": "ultravox", "class": "speech_recognition", "profile": "workload-oracle", - "cadences": ["manual-full"], + "cadences": ["llama-bump", "manual-full"], "artifact": {"repo": "ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF", "revision": "5390c7c41cbd6f261f7f205fc0c5ae61bbdca650", "files": ["Llama-3.2-1B-Instruct-Q8_0.gguf"], "file_integrity": {"Llama-3.2-1B-Instruct-Q8_0.gguf": {"size_bytes": 1321083008, "blob_id": "432f310a77f4650a88d0fd59ecdd7cebed8d684bafea53cbff0473542964f0c3"}}, "selector": "Q8_0"}, "mmproj_artifact": {"repo": "ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF", "revision": "5390c7c41cbd6f261f7f205fc0c5ae61bbdca650", "files": ["mmproj-ultravox-v0_5-llama-3_2-1b-f16.gguf"], "file_integrity": {"mmproj-ultravox-v0_5-llama-3_2-1b-f16.gguf": {"size_bytes": 1371123616, "blob_id": "b34dde1835752949d6b960528269af93c92fec91c61ea0534fcc73f96c1ed8b2"}}, "selector": "F16"}, "evidence": {"fixture": "ci/llama-canary/fixtures/audio-smoke.wav", "comparison": "identical normalized transcript against pinned llama-server; fixture is unlabeled"}, diff --git a/ci/model-artifacts/manifests/competitive-benchmark.json b/ci/model-artifacts/manifests/competitive-benchmark.json index 9a70a8f56a..d964cae23e 100644 --- a/ci/model-artifacts/manifests/competitive-benchmark.json +++ b/ci/model-artifacts/manifests/competitive-benchmark.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "competitive-benchmark", - "registry_sha256": "7391ea78891c52d41a39d77a2d6fe18b9630946cdbf28009f6a761ebb18a3bc4", + "registry_sha256": "70665c803b4d65dc6e2373314dce4b928698c3836774b52504655f8fac778614", "artifacts": [ { "id": "family-llama", diff --git a/ci/model-artifacts/manifests/hf-download-smoke.json b/ci/model-artifacts/manifests/hf-download-smoke.json index 0d04439c2b..25163fedaf 100644 --- a/ci/model-artifacts/manifests/hf-download-smoke.json +++ b/ci/model-artifacts/manifests/hf-download-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "hf-download-smoke", - "registry_sha256": "7391ea78891c52d41a39d77a2d6fe18b9630946cdbf28009f6a761ebb18a3bc4", + "registry_sha256": "70665c803b4d65dc6e2373314dce4b928698c3836774b52504655f8fac778614", "artifacts": [ { "id": "smollm2-q4-download", diff --git a/ci/model-artifacts/manifests/openai-smoke.json b/ci/model-artifacts/manifests/openai-smoke.json index 45a7d016db..22c3a2e367 100644 --- a/ci/model-artifacts/manifests/openai-smoke.json +++ b/ci/model-artifacts/manifests/openai-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "openai-smoke", - "registry_sha256": "7391ea78891c52d41a39d77a2d6fe18b9630946cdbf28009f6a761ebb18a3bc4", + "registry_sha256": "70665c803b4d65dc6e2373314dce4b928698c3836774b52504655f8fac778614", "artifacts": [ { "id": "smollm2-q4-download", diff --git a/ci/model-artifacts/manifests/product-integration-smoke.json b/ci/model-artifacts/manifests/product-integration-smoke.json index c8f426210f..06d1d0073b 100644 --- a/ci/model-artifacts/manifests/product-integration-smoke.json +++ b/ci/model-artifacts/manifests/product-integration-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "product-integration-smoke", - "registry_sha256": "7391ea78891c52d41a39d77a2d6fe18b9630946cdbf28009f6a761ebb18a3bc4", + "registry_sha256": "70665c803b4d65dc6e2373314dce4b928698c3836774b52504655f8fac778614", "artifacts": [ { "id": "family-granite-hybrid", diff --git a/ci/model-artifacts/manifests/product-smoke.json b/ci/model-artifacts/manifests/product-smoke.json index 05f1778a8c..9bdc932371 100644 --- a/ci/model-artifacts/manifests/product-smoke.json +++ b/ci/model-artifacts/manifests/product-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "product-smoke", - "registry_sha256": "7391ea78891c52d41a39d77a2d6fe18b9630946cdbf28009f6a761ebb18a3bc4", + "registry_sha256": "70665c803b4d65dc6e2373314dce4b928698c3836774b52504655f8fac778614", "artifacts": [ { "id": "smollm2-q8-inference", diff --git a/ci/model-artifacts/manifests/radix-cache.json b/ci/model-artifacts/manifests/radix-cache.json index 21c7692c4d..a9f0a6c4ac 100644 --- a/ci/model-artifacts/manifests/radix-cache.json +++ b/ci/model-artifacts/manifests/radix-cache.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "radix-cache", - "registry_sha256": "7391ea78891c52d41a39d77a2d6fe18b9630946cdbf28009f6a761ebb18a3bc4", + "registry_sha256": "70665c803b4d65dc6e2373314dce4b928698c3836774b52504655f8fac778614", "artifacts": [ { "id": "family-deepseek2", diff --git a/ci/model-artifacts/manifests/safetensors-runtime-smoke.json b/ci/model-artifacts/manifests/safetensors-runtime-smoke.json index f548dee6d3..f59e8fd571 100644 --- a/ci/model-artifacts/manifests/safetensors-runtime-smoke.json +++ b/ci/model-artifacts/manifests/safetensors-runtime-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "safetensors-runtime-smoke", - "registry_sha256": "7391ea78891c52d41a39d77a2d6fe18b9630946cdbf28009f6a761ebb18a3bc4", + "registry_sha256": "70665c803b4d65dc6e2373314dce4b928698c3836774b52504655f8fac778614", "artifacts": [ { "id": "smollm2-safetensors", diff --git a/ci/model-artifacts/manifests/scripted-binary-smoke.json b/ci/model-artifacts/manifests/scripted-binary-smoke.json index 2b2dcea79f..3bfbcb8639 100644 --- a/ci/model-artifacts/manifests/scripted-binary-smoke.json +++ b/ci/model-artifacts/manifests/scripted-binary-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "scripted-binary-smoke", - "registry_sha256": "7391ea78891c52d41a39d77a2d6fe18b9630946cdbf28009f6a761ebb18a3bc4", + "registry_sha256": "70665c803b4d65dc6e2373314dce4b928698c3836774b52504655f8fac778614", "artifacts": [ { "id": "smollm2-q8-inference", diff --git a/ci/model-artifacts/manifests/sdk-smoke.json b/ci/model-artifacts/manifests/sdk-smoke.json index 53f8fcf782..2e0afeddfc 100644 --- a/ci/model-artifacts/manifests/sdk-smoke.json +++ b/ci/model-artifacts/manifests/sdk-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "sdk-smoke", - "registry_sha256": "7391ea78891c52d41a39d77a2d6fe18b9630946cdbf28009f6a761ebb18a3bc4", + "registry_sha256": "70665c803b4d65dc6e2373314dce4b928698c3836774b52504655f8fac778614", "artifacts": [ { "id": "smollm2-q8-inference", diff --git a/ci/model-artifacts/manifests/skippy-ci-smoke.json b/ci/model-artifacts/manifests/skippy-ci-smoke.json index af5b01c25f..c3ef89206e 100644 --- a/ci/model-artifacts/manifests/skippy-ci-smoke.json +++ b/ci/model-artifacts/manifests/skippy-ci-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "skippy-ci-smoke", - "registry_sha256": "7391ea78891c52d41a39d77a2d6fe18b9630946cdbf28009f6a761ebb18a3bc4", + "registry_sha256": "70665c803b4d65dc6e2373314dce4b928698c3836774b52504655f8fac778614", "artifacts": [ { "id": "family-qwen3-dense", diff --git a/ci/model-artifacts/manifests/skippy-correctness.json b/ci/model-artifacts/manifests/skippy-correctness.json index c3cf6b030f..bc06d29a91 100644 --- a/ci/model-artifacts/manifests/skippy-correctness.json +++ b/ci/model-artifacts/manifests/skippy-correctness.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "skippy-correctness", - "registry_sha256": "7391ea78891c52d41a39d77a2d6fe18b9630946cdbf28009f6a761ebb18a3bc4", + "registry_sha256": "70665c803b4d65dc6e2373314dce4b928698c3836774b52504655f8fac778614", "artifacts": [ { "id": "qwen3-q8-correctness", diff --git a/ci/model-artifacts/manifests/skippy-parity.json b/ci/model-artifacts/manifests/skippy-parity.json index 8222387f94..6c966872b2 100644 --- a/ci/model-artifacts/manifests/skippy-parity.json +++ b/ci/model-artifacts/manifests/skippy-parity.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "skippy-parity", - "registry_sha256": "7391ea78891c52d41a39d77a2d6fe18b9630946cdbf28009f6a761ebb18a3bc4", + "registry_sha256": "70665c803b4d65dc6e2373314dce4b928698c3836774b52504655f8fac778614", "artifacts": [ { "id": "family-deepseek2", diff --git a/ci/model-artifacts/registry.json b/ci/model-artifacts/registry.json index 8a8e2a1610..85f8a109ce 100644 --- a/ci/model-artifacts/registry.json +++ b/ci/model-artifacts/registry.json @@ -4012,6 +4012,7 @@ "llama-family-certification" ], "cadences": [ + "llama-bump", "manual-full" ], "capability_tags": [ @@ -4034,6 +4035,7 @@ "class": "embedding", "profile": "workload-oracle", "cadences": [ + "llama-bump", "manual-full" ], "execution": { @@ -4062,6 +4064,7 @@ "llama-family-certification" ], "cadences": [ + "llama-bump", "manual-full" ], "capability_tags": [ @@ -4084,6 +4087,7 @@ "class": "rerank", "profile": "workload-oracle", "cadences": [ + "llama-bump", "manual-full" ], "execution": { @@ -4112,6 +4116,7 @@ "llama-family-certification" ], "cadences": [ + "llama-bump", "manual-full" ], "capability_tags": [ @@ -4134,6 +4139,7 @@ "class": "encoder_decoder", "profile": "workload-oracle", "cadences": [ + "llama-bump", "manual-full" ], "execution": { @@ -4162,6 +4168,7 @@ "llama-family-certification" ], "cadences": [ + "llama-bump", "manual-full" ], "capability_tags": [ @@ -4184,6 +4191,7 @@ "class": "ocr", "profile": "workload-oracle", "cadences": [ + "llama-bump", "manual-full" ], "execution": { @@ -4225,6 +4233,7 @@ "llama-family-certification" ], "cadences": [ + "llama-bump", "manual-full" ], "capability_tags": [ @@ -4247,6 +4256,7 @@ "class": "speech_synthesis", "profile": "workload-oracle", "cadences": [ + "llama-bump", "manual-full" ], "execution": { @@ -4288,6 +4298,7 @@ "llama-family-certification" ], "cadences": [ + "llama-bump", "manual-full" ], "capability_tags": [ @@ -4310,6 +4321,7 @@ "class": "speech_recognition", "profile": "workload-oracle", "cadences": [ + "llama-bump", "manual-full" ], "execution": { diff --git a/just/skippy.just b/just/skippy.just index 43fa0b683a..f5dff0a53f 100644 --- a/just/skippy.just +++ b/just/skippy.just @@ -44,6 +44,11 @@ skippy-quantize-standalone-release-build backend="cpu": LLAMA_STAGE_BACKEND="{{ backend }}" LLAMA_STAGE_LINK_MODE=static scripts/build-llama.sh LLAMA_STAGE_BACKEND="{{ backend }}" LLAMA_STAGE_LINK_MODE=static just with-lld cargo build --release --locked -p skippy-quantize --no-default-features +# Build isolated CPU workload certification producers (not a shipping runtime). +[unix] +skippy-workload-oracles-build output: + bash scripts/skippy-workload-oracles-build.sh "{{ output }}" + # Build and execute the opt-in private native Skippy application probes. [unix] skippy-native-tests backend="cpu": diff --git a/scripts/check-skippy-workload-candidate.py b/scripts/check-skippy-workload-candidate.py index 4643125efb..7748612bf3 100644 --- a/scripts/check-skippy-workload-candidate.py +++ b/scripts/check-skippy-workload-candidate.py @@ -4,8 +4,74 @@ from __future__ import annotations import argparse +import hashlib +import json import os from pathlib import Path +import subprocess + +ROOT = Path(__file__).resolve().parents[1] + + +def file_hash(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def source_identity(root: Path = ROOT) -> dict[str, str]: + def git(*args: str) -> bytes: + return subprocess.check_output(["git", "-C", str(root), *args]) + + digest = hashlib.sha256(git("diff", "--binary", "HEAD", "--")) + # Bind new source files as well as staged/unstaged tracked changes. Build + # outputs are ignored by Git and cannot change this identity during a run. + for name in sorted(git("ls-files", "--others", "--exclude-standard", "-z").split(b"\0")): + if name: + digest.update(name + b"\0") + digest.update(file_hash(root / os.fsdecode(name)).encode("ascii")) + return {"head": git("rev-parse", "HEAD").decode().strip(), "worktree_sha256": digest.hexdigest()} + + +def producer_files(binary: Path, build_dir: Path, test_binary: Path) -> dict[str, Path]: + return { + "candidate": binary, + "test_binary": test_binary, + "model_package": binary.parent / "skippy-model-package", + "correctness": binary.parent / "skippy-correctness", + "native_stamp": build_dir / ".mesh-llm-build-stamp", + **{name: build_dir / "bin" / name for name in ("llama-server", "llama-completion", "llama-tts")}, + } + + +def write_producer(output: Path, binary: Path, build_dir: Path, test_binary: Path, source_snapshot: Path) -> None: + source = source_identity() + if source != json.loads(source_snapshot.read_text(encoding="utf-8")): + raise RuntimeError("repository source changed while building workload producers") + check_candidate(binary, build_dir) + check_candidate(test_binary, build_dir) + files = producer_files(binary, build_dir, test_binary) + payload = { + "schema_version": 1, + "source": source, + "files": {name: {"path": str(path.resolve()), "sha256": file_hash(path)} for name, path in files.items()}, + } + output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def verify_producer(manifest: Path, binary: Path, build_dir: Path) -> None: + payload = json.loads(manifest.read_text(encoding="utf-8")) + if payload.get("schema_version") != 1 or payload.get("source") != source_identity(): + raise RuntimeError("workload producer does not match the current repository head and worktree") + records = payload["files"] + files = producer_files(binary, build_dir, Path(records["test_binary"]["path"])) + for name, path in files.items(): + record = records[name] + if record["path"] != str(path.resolve()) or record["sha256"] != file_hash(path): + raise RuntimeError(f"workload producer artifact changed: {name}") + check_candidate(files["test_binary"], build_dir) def check_candidate(binary: Path, build_dir: Path) -> None: @@ -23,10 +89,27 @@ def check_candidate(binary: Path, build_dir: Path) -> None: def main() -> None: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--candidate-binary", required=True, type=Path) - parser.add_argument("--native-build-dir", required=True, type=Path) + parser.add_argument("--candidate-binary", type=Path) + parser.add_argument("--native-build-dir", type=Path) + producer = parser.add_mutually_exclusive_group() + producer.add_argument("--write-producer", type=Path) + producer.add_argument("--producer-manifest", type=Path) + parser.add_argument("--test-binary", type=Path) + parser.add_argument("--source-snapshot", type=Path) + parser.add_argument("--write-source-snapshot", type=Path) args = parser.parse_args() + if args.write_source_snapshot: + args.write_source_snapshot.write_text(json.dumps(source_identity()) + "\n", encoding="utf-8") + return + if not args.candidate_binary or not args.native_build_dir: + parser.error("--candidate-binary and --native-build-dir are required") check_candidate(args.candidate_binary, args.native_build_dir) + if args.write_producer: + if not args.test_binary or not args.source_snapshot: + parser.error("--write-producer requires --test-binary and --source-snapshot") + write_producer(args.write_producer, args.candidate_binary, args.native_build_dir, args.test_binary, args.source_snapshot) + elif args.producer_manifest: + verify_producer(args.producer_manifest, args.candidate_binary, args.native_build_dir) if __name__ == "__main__": diff --git a/scripts/llama-canary-agent-repair.sh b/scripts/llama-canary-agent-repair.sh index c9764d94b9..9dbdf076e2 100755 --- a/scripts/llama-canary-agent-repair.sh +++ b/scripts/llama-canary-agent-repair.sh @@ -388,9 +388,16 @@ run_full_build() { || return 1 run_verification_logged "Skippy smoke tests" "$BUILD_LOG" \ scripts/skippy-ci-smoke.sh || return 1 + run_verification_logged "pinned CPU workload oracles and candidate" "$BUILD_LOG" \ + just skippy-workload-oracles-build "${LLAMA_STAGE_BUILD_DIR:?}-workloads" || return 1 } run_certification() { + local setting + local workload_env=() + while IFS= read -r setting; do + workload_env+=("$setting") + done < <(bash scripts/skippy-workload-oracles-build.sh --print-env "${LLAMA_STAGE_BUILD_DIR:?}-workloads") : > "$CERTIFY_LOG" echo "trusted candidate gate: certify" | tee -a "$CERTIFY_LOG" run_verification_logged "parity manifest validation" "$CERTIFY_LOG" \ @@ -412,6 +419,7 @@ run_certification() { scripts/skippy-canary-live-matrix.sh --prepare || return 1 run_verification_logged "full supported-family certification" "$CERTIFY_LOG" env \ FAMILY_BATTERY_RUN_ID="$FAMILY_BATTERY_RUN_ID" \ + "${workload_env[@]}" \ scripts/skippy-family-battery.sh --skip-build --plan "$PLAN_PATH" } diff --git a/scripts/skippy-family-battery.sh b/scripts/skippy-family-battery.sh index cf0aa191ca..1949c2b658 100755 --- a/scripts/skippy-family-battery.sh +++ b/scripts/skippy-family-battery.sh @@ -161,6 +161,9 @@ if [[ -n "$FAMILY_FILTER" && ! "$FAMILY_FILTER" =~ ^[a-zA-Z0-9._-]+(,[a-zA-Z0-9. fi mkdir -p "$MODEL_SCAN_DIR" "$PREFLIGHT_DIR" "$CERT_DIR" +if [[ -n "${SKIPPY_WORKLOAD_PRODUCER_MANIFEST:-}" ]]; then + cp "$SKIPPY_WORKLOAD_PRODUCER_MANIFEST" "$ARTIFACT_DIR/workload-producer.json" +fi : > "$RESULTS_JSONL" printf 'family\tmodel_id\tsource_revision\tmodel_path\tmtp_layers\n' > "$NATIVE_MTP_MODELS_TSV" printf 'family|class|repo|source_revision|file|selector|sweep_period|layer_end|notes|target_path|draft_repo|draft_revision|draft_file|draft_path|native_mtp|model_size_bytes|mtp_layers|activation_width|startup_timeout_secs|lane_csv|mmproj_repo|mmproj_revision|mmproj_file|mmproj_path\n' > "$RESOLVED_MANIFEST" @@ -942,7 +945,7 @@ run_workload_certify() { --evidence "$cert_run_dir/workload-oracle-evidence.json" \ --class "$model_class" --smoke-lane "$smoke_lane" --oracle-lane "$oracle_lane" \ --model-id "$model_id" --model-path "$target" \ - --candidate-executable "$ROOT/target/debug/skippy-server" \ + --candidate-executable "${SKIPPY_WORKLOAD_CANDIDATE_BIN_DIR:-$ROOT/target/debug}/skippy-server" \ --oracle-executable "$oracle_executable" \ --pinned-patch-sha "$(python3 "$ROOT/scripts/llama-oracle-source.py")") if [[ -n "$mmproj" ]]; then diff --git a/scripts/skippy-workload-certify.sh b/scripts/skippy-workload-certify.sh index b4bcac5eef..30961fd910 100755 --- a/scripts/skippy-workload-certify.sh +++ b/scripts/skippy-workload-certify.sh @@ -20,7 +20,7 @@ usage: scripts/skippy-workload-certify.sh --class CLASS --lane LANE --model-path PATH --model-id ID --work-dir PATH [--projector-path PATH] [--oracle-server PATH] [--oracle-completion PATH] [--oracle-tts PATH] [--require-oracle] # fail closed unless the class-appropriate oracle is selected - [--skip-build] # skips the candidate build only for smoke-only runs + [--skip-build] # oracle runs require a prebuilt SKIPPY_WORKLOAD_PRODUCER_MANIFEST EOF } @@ -78,7 +78,10 @@ if (( ORACLE_REQUIRED == 1 )) && [[ -z "$ORACLE_SERVER" && -z "$ORACLE_COMPLETIO echo "certified workload requires a class-appropriate local-monolithic oracle" >&2 exit 1 fi -CANDIDATE_BUILD_DIR="${LLAMA_STAGE_BUILD_DIR:-$(LLAMA_STAGE_BACKEND="${LLAMA_STAGE_BACKEND:-cpu}" LLAMA_STAGE_LINK_MODE=static "$ROOT/scripts/build-llama.sh" --print-build-dir)}" +CANDIDATE_BUILD_DIR="${SKIPPY_WORKLOAD_NATIVE_BUILD_DIR:-${LLAMA_STAGE_BUILD_DIR:-$(LLAMA_STAGE_BACKEND="${LLAMA_STAGE_BACKEND:-cpu}" LLAMA_STAGE_LINK_MODE=static "$ROOT/scripts/build-llama.sh" --print-build-dir)}}" +CANDIDATE_BIN_DIR="${SKIPPY_WORKLOAD_CANDIDATE_BIN_DIR:-$ROOT/target/debug}" +PRODUCER_MANIFEST="${SKIPPY_WORKLOAD_PRODUCER_MANIFEST:-}" +TEST_COMMAND=(cargo test --manifest-path "$ROOT/Cargo.toml" -p skippy-server --lib) require_pinned_cpu_oracle() { local executable="$1" expected_name="$2" cmake_option="$3" local build_dir stamp candidate_build_dir candidate_stamp patched_sha @@ -140,6 +143,9 @@ LAYER_END="$(jq -r '.layer_count' <<<"$DIMENSIONS")" MODEL_SHA256="$(shasum -a 256 "$MODEL_PATH" | awk '{print $1}')" N_GPU_LAYERS="${SKIPPY_WORKLOAD_N_GPU_LAYERS:-0}" BACKEND="${LLAMA_STAGE_BACKEND:-cpu}" +if [[ -n "$PRODUCER_MANIFEST" ]]; then + BACKEND=cpu +fi if [[ ( -n "$ORACLE_SERVER" || -n "$ORACLE_COMPLETION" || -n "$ORACLE_TTS" ) && ( "$N_GPU_LAYERS" != "0" || "$BACKEND" != "cpu" ) ]]; then echo "local-monolithic comparison requires CPU-only candidate execution" >&2 @@ -151,17 +157,24 @@ if [[ -n "$ORACLE_SERVER" || -n "$ORACLE_COMPLETION" || -n "$ORACLE_TTS" ]]; the export LLAMA_STAGE_BUILD_DIR="$CANDIDATE_BUILD_DIR" fi -# An oracle result must never be based on a stale Rust executable. Cargo tracks -# both Rust sources and the native archives, so always refresh the candidate -# when comparing against a monolithic reference, even for battery runs that -# prebuilt binaries and passed --skip-build. -if (( SKIP_BUILD == 0 )) || [[ -n "$ORACLE_SERVER" || -n "$ORACLE_COMPLETION" || -n "$ORACLE_TTS" ]]; then +# The canary explicitly produces a CPU candidate and test binary separately +# from its Metal lane. Consume that immutable, source-bound closure without +# rebuilding or changing the other family lanes' native/Rust outputs. +if [[ -n "$PRODUCER_MANIFEST" ]]; then + python3 "$ROOT/scripts/check-skippy-workload-candidate.py" \ + --candidate-binary "$CANDIDATE_BIN_DIR/skippy-server" \ + --native-build-dir "$CANDIDATE_BUILD_DIR" --producer-manifest "$PRODUCER_MANIFEST" + TEST_COMMAND=("$(jq -er '.files.test_binary.path' "$PRODUCER_MANIFEST")") +elif (( SKIP_BUILD == 0 )); then LLAMA_STAGE_BUILD_DIR="$CANDIDATE_BUILD_DIR" \ cargo build -p skippy-server +elif [[ -n "$ORACLE_SERVER" || -n "$ORACLE_COMPLETION" || -n "$ORACLE_TTS" ]]; then + echo "--skip-build oracle certification requires a source-bound workload producer manifest" >&2 + exit 1 fi if [[ -n "$ORACLE_SERVER" || -n "$ORACLE_COMPLETION" || -n "$ORACLE_TTS" ]]; then python3 "$ROOT/scripts/check-skippy-workload-candidate.py" \ - --candidate-binary "$ROOT/target/debug/skippy-server" \ + --candidate-binary "$CANDIDATE_BIN_DIR/skippy-server" \ --native-build-dir "$CANDIDATE_BUILD_DIR" fi @@ -171,6 +184,12 @@ case "$MODEL_CLASS" in speech_recognition) MEDIA_PATH="$ROOT/ci/llama-canary/fixtures/audio-smoke.wav" ;; esac +TEST_FILTER=frontend::tests::non_chat::real_non_chat_class_smoke_when_fixture_is_set +if [[ -n "$PRODUCER_MANIFEST" ]]; then + TEST_COMMAND+=("$TEST_FILTER" --nocapture --exact --test-threads=1) +else + TEST_COMMAND+=("$TEST_FILTER" -- --nocapture --exact --test-threads=1) +fi env \ SKIPPY_WORKLOAD_CLASS="$MODEL_CLASS" \ SKIPPY_WORKLOAD_MODEL="$MODEL_PATH" \ @@ -182,9 +201,7 @@ env \ SKIPPY_WORKLOAD_MAX_TOKENS="${SKIPPY_WORKLOAD_MAX_TOKENS:-32}" \ SKIPPY_WORKLOAD_N_GPU_LAYERS="$N_GPU_LAYERS" \ LLAMA_STAGE_BACKEND="$BACKEND" \ - cargo test --manifest-path "$ROOT/Cargo.toml" -p skippy-server --lib \ - frontend::tests::non_chat::real_non_chat_class_smoke_when_fixture_is_set \ - -- --nocapture --exact --test-threads=1 + "${TEST_COMMAND[@]}" PORT="${SKIPPY_WORKLOAD_OPENAI_PORT:-19337}" CONFIG_PATH="$WORK_DIR/stage-openai.json" @@ -225,7 +242,7 @@ PY SERVER_LOG="$WORK_DIR/workload-openai-server.log" LLAMA_STAGE_BACKEND="$BACKEND" \ - "$ROOT/target/debug/skippy-server" serve-openai \ + "$CANDIDATE_BIN_DIR/skippy-server" serve-openai \ --config "$CONFIG_PATH" \ --bind-addr "127.0.0.1:$PORT" \ --telemetry-level off \ @@ -346,7 +363,7 @@ if [[ -n "$ORACLE_SERVER" || -n "$ORACLE_COMPLETION" || -n "$ORACLE_TTS" ]]; the --smoke-lane "$EXPECTED_LANE" --model-id "$MODEL_ID" --model-sha256 "$MODEL_SHA256" - --candidate-executable "$ROOT/target/debug/skippy-server" + --candidate-executable "$CANDIDATE_BIN_DIR/skippy-server" --oracle-executable "$ORACLE_EXECUTABLE" --pinned-patch-sha "$(python3 "$ROOT/scripts/llama-oracle-source.py")" --work-dir "$WORK_DIR") diff --git a/scripts/skippy-workload-oracles-build.sh b/scripts/skippy-workload-oracles-build.sh new file mode 100644 index 0000000000..bde64a79f2 --- /dev/null +++ b/scripts/skippy-workload-oracles-build.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Explicit producer for the CPU workload candidate and its monolithic oracles. +# Called through just; never touches the canary's Metal/native or Rust outputs. +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PRINT_ENV=0 +if [[ "${1:-}" == "--print-env" ]]; then + PRINT_ENV=1 + shift +fi +if [[ $# != 1 || "$1" != /* || "$1" == *$'\n'* || "$1" == *$'\r'* ]]; then + echo "usage: skippy-workload-oracles-build.sh [--print-env] ABSOLUTE_BUILD_ROOT" >&2 + exit 1 +fi +BUILD_ROOT="$1" +NATIVE_DIR="$BUILD_ROOT/native" +CARGO_DIR="$BUILD_ROOT/cargo" +MANIFEST="$BUILD_ROOT/producer.json" +if (( PRINT_ENV == 1 )); then + printf '%s\n' \ + "SKIPPY_WORKLOAD_ORACLE_SERVER=$NATIVE_DIR/bin/llama-server" \ + "SKIPPY_WORKLOAD_ORACLE_COMPLETION=$NATIVE_DIR/bin/llama-completion" \ + "SKIPPY_WORKLOAD_ORACLE_TTS=$NATIVE_DIR/bin/llama-tts" \ + "SKIPPY_WORKLOAD_NATIVE_BUILD_DIR=$NATIVE_DIR" \ + "SKIPPY_WORKLOAD_CANDIDATE_BIN_DIR=$CARGO_DIR/debug" \ + "SKIPPY_WORKLOAD_PRODUCER_MANIFEST=$MANIFEST" + exit 0 +fi +cd "$ROOT" +mkdir -p "$BUILD_ROOT" +python3 scripts/check-skippy-workload-candidate.py --write-source-snapshot "$BUILD_ROOT/source.json" +python3 scripts/llama-oracle-source.py +export LLAMA_STAGE_BACKEND=cpu LLAMA_STAGE_LINK_MODE=static +export LLAMA_BUILD_DIR="$NATIVE_DIR" LLAMA_STAGE_BUILD_DIR="$NATIVE_DIR" +export LLAMA_STAGE_WORKLOAD_ORACLE=ON LLAMA_STAGE_UPSTREAM_TESTS=OFF +export LLAMA_STAGE_FULL_REPLAY=OFF LLAMA_STAGE_BUILD_TESTS=OFF +export CARGO_TARGET_DIR="$CARGO_DIR" +native_args=() +if [[ "$(uname -s)" == Darwin ]]; then + native_args+=(-DCMAKE_OSX_ARCHITECTURES=arm64) +fi +scripts/build-llama.sh "${native_args[@]}" +just with-lld cargo build --locked -p skippy-server -p skippy-model-package -p skippy-correctness +just with-lld cargo test --locked -p skippy-server --lib --no-run --message-format=json > "$BUILD_ROOT/test-artifacts.jsonl" +test_binary="$(jq -rs '[.[] | select(.reason == "compiler-artifact" and .profile.test == true and .target.name == "skippy_server" and .executable != null) | .executable] | unique | if length == 1 then .[0] else error("expected one skippy-server library test binary") end' "$BUILD_ROOT/test-artifacts.jsonl")" +python3 scripts/check-skippy-workload-candidate.py \ + --candidate-binary "$CARGO_DIR/debug/skippy-server" \ + --native-build-dir "$NATIVE_DIR" \ + --test-binary "$test_binary" --write-producer "$MANIFEST" --source-snapshot "$BUILD_ROOT/source.json" diff --git a/scripts/tests/test_check_skippy_workload_candidate.py b/scripts/tests/test_check_skippy_workload_candidate.py index 9e5772416f..2df003351f 100644 --- a/scripts/tests/test_check_skippy_workload_candidate.py +++ b/scripts/tests/test_check_skippy_workload_candidate.py @@ -1,18 +1,59 @@ from __future__ import annotations import os +import importlib.util +import json from pathlib import Path import subprocess import sys import tempfile import unittest +from unittest import mock ROOT = Path(__file__).resolve().parents[2] CHECK = ROOT / "scripts" / "check-skippy-workload-candidate.py" +SPEC = importlib.util.spec_from_file_location("workload_candidate", CHECK) +assert SPEC and SPEC.loader +CANDIDATE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(CANDIDATE) class CandidateBuildFreshnessTests(unittest.TestCase): + def test_producer_binds_source_native_stamp_and_every_executable(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + native = root / "native" + (native / "bin").mkdir(parents=True) + stamp = native / ".mesh-llm-build-stamp" + stamp.write_text("cpu native fixture") + os.utime(stamp, ns=(1, 1)) + binary, test_binary = root / "skippy-server", root / "skippy-tests" + files = CANDIDATE.producer_files(binary, native, test_binary) + for name, path in files.items(): + if name != "native_stamp": + path.write_text(name) + path.chmod(0o755) + source = {"head": "a" * 40, "worktree_sha256": "b" * 64} + snapshot = root / "source.json" + snapshot.write_text(json.dumps(source)) + manifest = root / "producer.json" + with mock.patch.object(CANDIDATE, "source_identity", return_value=source): + CANDIDATE.write_producer(manifest, binary, native, test_binary, snapshot) + CANDIDATE.verify_producer(manifest, binary, native) + for name, path in files.items(): + with self.subTest(artifact=name): + original = path.read_bytes() + path.write_bytes(original + b"tampered") + with self.assertRaisesRegex(RuntimeError, "artifact changed"): + CANDIDATE.verify_producer(manifest, binary, native) + path.write_bytes(original) + source["head"] = "c" * 40 + with self.assertRaisesRegex(RuntimeError, "current repository head"): + CANDIDATE.verify_producer(manifest, binary, native) + with self.assertRaisesRegex(RuntimeError, "source changed while building"): + CANDIDATE.write_producer(manifest, binary, native, test_binary, snapshot) + def _check(self, binary: Path, build_dir: Path) -> subprocess.CompletedProcess[str]: return subprocess.run( [ diff --git a/scripts/tests/test_justfile_layout.py b/scripts/tests/test_justfile_layout.py index fd8fa650a1..bce068fd84 100644 --- a/scripts/tests/test_justfile_layout.py +++ b/scripts/tests/test_justfile_layout.py @@ -37,6 +37,7 @@ "bench-corpus", "competitive-benchmark-build", "family-certify", "metrics-server", "metrics-server-build", "skippy-native-full-replay", "skippy-native-tests", "skippy-openai-smoke", + "skippy-workload-oracles-build", "skippy-quantize-build", "skippy-quantize-release-build", "skippy-quantize-standalone-build", "skippy-quantize-standalone-release-build", "skippy-wan-lab-build-bins", diff --git a/scripts/tests/test_plan_family_battery.py b/scripts/tests/test_plan_family_battery.py index f942e6a383..9a5454d813 100644 --- a/scripts/tests/test_plan_family_battery.py +++ b/scripts/tests/test_plan_family_battery.py @@ -178,7 +178,7 @@ def test_checked_in_policy_resolves_all_certified_models(self) -> None: self.assertEqual("workload-oracle", model["profile"]) self.assertEqual("certified", model["certification_status"]) self.assertEqual("local-monolithic", model["oracle"]) - self.assertEqual(["manual-full"], model["cadences"]) + self.assertEqual(["llama-bump", "manual-full"], model["cadences"]) self.assertEqual("disabled", model["execution"]["speculative_policy"]) self.assertEqual(0, model["execution"]["boundary_sweep_period"]) self.assertEqual(0, model["execution"]["mtp_layers"]) diff --git a/scripts/tests/test_skippy_workload_oracles_build.py b/scripts/tests/test_skippy_workload_oracles_build.py new file mode 100644 index 0000000000..ae08d53708 --- /dev/null +++ b/scripts/tests/test_skippy_workload_oracles_build.py @@ -0,0 +1,52 @@ +"""CPU producer contract shared by unchanged and changed-pin canaries.""" +from pathlib import Path +import json +import subprocess +import unittest + +ROOT = Path(__file__).resolve().parents[2] +PRODUCER = ROOT / "scripts/skippy-workload-oracles-build.sh" + + +class WorkloadOracleProducerTests(unittest.TestCase): + def test_environment_is_deterministic_and_does_not_override_metal_outputs(self) -> None: + result = subprocess.run( + ["bash", str(PRODUCER), "--print-env", "/tmp/canary with spaces"], + text=True, capture_output=True, check=True, + ) + values = dict(line.split("=", 1) for line in result.stdout.splitlines()) + self.assertEqual(6, len(values)) + self.assertTrue(all(name.startswith("SKIPPY_WORKLOAD_") for name in values)) + for suffix, executable in [("SERVER", "llama-server"), ("COMPLETION", "llama-completion"), ("TTS", "llama-tts")]: + self.assertEqual(f"/tmp/canary with spaces/native/bin/{executable}", values[f"SKIPPY_WORKLOAD_ORACLE_{suffix}"]) + self.assertEqual("/tmp/canary with spaces/cargo/debug", values["SKIPPY_WORKLOAD_CANDIDATE_BIN_DIR"]) + + def test_rejects_relative_or_environment_injection_paths(self) -> None: + for path in ["relative", "/tmp/line\nGH_TOKEN=bad", "/tmp/line\rnext"]: + result = subprocess.run(["bash", str(PRODUCER), "--print-env", path], capture_output=True) + self.assertNotEqual(0, result.returncode) + + def test_both_canary_paths_build_then_export_the_same_cpu_producers(self) -> None: + workflow = (ROOT / ".github/workflows/llama-upstream-canary.yml").read_text() + repair = (ROOT / "scripts/llama-canary-agent-repair.sh").read_text() + for text in [workflow, repair]: + self.assertIn("just skippy-workload-oracles-build", text) + self.assertIn("skippy-workload-oracles-build.sh --print-env", text) + self.assertLess(workflow.index("name: Build pinned CPU workload"), workflow.index("name: Supported-families certification battery")) + producer = PRODUCER.read_text() + for contract in ["LLAMA_STAGE_BACKEND=cpu", "LLAMA_STAGE_LINK_MODE=static", "LLAMA_STAGE_WORKLOAD_ORACLE=ON", "CARGO_TARGET_DIR=", "--no-run --message-format=json", "--write-producer", "--source-snapshot"]: + self.assertIn(contract, producer) + consumer = (ROOT / "scripts/skippy-workload-certify.sh").read_text() + self.assertIn("--producer-manifest", consumer) + self.assertIn('TEST_COMMAND=("$(jq -er', consumer) + + def test_all_six_workload_rows_guard_pin_advances_and_forced_certification(self) -> None: + for cadence in ["llama-bump", "manual-full"]: + result = subprocess.run( + [str(ROOT / "scripts/plan-family-battery.py"), "--cadence", cadence], + cwd=ROOT, text=True, capture_output=True, check=True, + ) + rows = [row for row in json.loads(result.stdout)["selected_models"] if row["class"] != "causal_generation"] + self.assertEqual(6, len(rows)) + self.assertEqual({"embedding", "rerank", "encoder_decoder", "ocr", "speech_synthesis", "speech_recognition"}, {row["class"] for row in rows}) + self.assertTrue(all(row["profile"] == "workload-oracle" for row in rows)) From b28ac30c86ba4832dba3bc0bca6defc9e61129dc Mon Sep 17 00:00:00 2001 From: IvGolovach <20299097+IvGolovach@users.noreply.github.com> Date: Sat, 12 Sep 2026 16:25:59 -0700 Subject: [PATCH 06/18] fix(skippy): allow safe serving listener restarts on Unix --- crates/skippy-server/src/http.rs | 43 ++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/crates/skippy-server/src/http.rs b/crates/skippy-server/src/http.rs index c0733c2de3..922708ef88 100644 --- a/crates/skippy-server/src/http.rs +++ b/crates/skippy-server/src/http.rs @@ -51,6 +51,13 @@ pub(crate) fn bind_serve_listener(bind_addr: SocketAddr) -> Result }; let socket = socket2::Socket::new(domain, socket2::Type::STREAM, Some(socket2::Protocol::TCP)) .with_context(|| format!("create serving socket for {bind_addr}"))?; + // Match Tokio's Unix listener semantics: a stopped server may rebind while + // its closed connections are in TIME_WAIT. Do not enable SO_REUSEPORT or + // Windows SO_REUSEADDR, which can permit sharing a live listener's port. + #[cfg(unix)] + socket + .set_reuse_address(true) + .context("enable serving listener address reuse")?; socket .bind(&bind_addr.into()) .with_context(|| format!("bind serving socket to {bind_addr}"))?; @@ -806,6 +813,42 @@ mod tests { use super::*; + #[cfg(unix)] + #[tokio::test] + async fn serving_listener_can_rebind_after_server_closes_connection() -> Result<()> { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + tokio::time::timeout(std::time::Duration::from_secs(5), async { + let listener = bind_serve_listener("127.0.0.1:0".parse()?)?; + let addr = listener.local_addr()?; + let mut client = tokio::net::TcpStream::connect(addr).await?; + let (mut accepted, _) = listener.accept().await?; + // The server actively closes, leaving its connection in TIME_WAIT. + accepted.shutdown().await?; + client.read_to_end(&mut Vec::new()).await?; + drop(accepted); + drop(client); + drop(listener); + let replacement = bind_serve_listener(addr)?; + assert_eq!(replacement.local_addr()?, addr); + Ok::<_, anyhow::Error>(()) + }) + .await + .context("serving listener restart timed out")? + } + + #[tokio::test] + async fn serving_listener_rejects_another_live_listener() -> Result<()> { + let listener = bind_serve_listener("127.0.0.1:0".parse()?)?; + let error = bind_serve_listener(listener.local_addr()?) + .expect_err("a second listener must not share the live port"); + assert_eq!( + error.downcast_ref::().unwrap().kind(), + std::io::ErrorKind::AddrInUse + ); + Ok(()) + } + fn stage_config_without_runtime() -> StageConfig { StageConfig { run_id: "run".to_owned(), From 95f301b1c4d305a6fb9b3854827e010a67e54e7d Mon Sep 17 00:00:00 2001 From: IvGolovach <20299097+IvGolovach@users.noreply.github.com> Date: Sat, 12 Sep 2026 16:49:04 -0700 Subject: [PATCH 07/18] test(skippy): align multimodal oracle execution contracts --- scripts/skippy-ocr-asr-oracle.py | 7 ++-- scripts/skippy-tts-oracle.py | 27 +++++++++++++-- scripts/skippy-workload-certify.sh | 1 + scripts/tests/test_skippy_ocr_asr_oracle.py | 12 ++++++- scripts/tests/test_skippy_tts_oracle.py | 37 ++++++++++++++++++++- 5 files changed, 77 insertions(+), 7 deletions(-) diff --git a/scripts/skippy-ocr-asr-oracle.py b/scripts/skippy-ocr-asr-oracle.py index aefbb859a3..b205c2e5bc 100644 --- a/scripts/skippy-ocr-asr-oracle.py +++ b/scripts/skippy-ocr-asr-oracle.py @@ -21,6 +21,7 @@ OCR_PROMPT = "Read all visible text. Return only the transcription." OCR_FIXTURE_TEXT = "MESH 42" ASR_PROMPT = "Transcribe audio to text" +ASR_MAX_TOKENS = 128 BOUNDARY = "mesh-llm-ocr-asr-oracle" NON_TRANSCRIPT_PREFIXES = ( "i can t fulfill", @@ -164,20 +165,22 @@ def compare_asr(candidate_url: str, oracle_url: str, model: str, audio: bytes, # llama-server's /audio/transcriptions substitutes its own default user # instruction. Compare the actual Skippy audio route with monolithic chat # using the exact instruction and media ordering that Skippy constructs. + # message_content_to_generation_text joins text/media parts with a newline; + # llama-server concatenates them directly, so preserve that separator here. candidate = request_multipart(candidate_url, "/audio/transcriptions", model, audio) reference = request_json(oracle_url, "/chat/completions", { "model": model, "messages": [{ "role": "user", "content": [ - {"type": "text", "text": ASR_PROMPT}, + {"type": "text", "text": ASR_PROMPT + "\n"}, {"type": "input_audio", "input_audio": { "data": base64.b64encode(audio).decode("ascii"), "format": "wav" }}, ], }], "temperature": 0.0, - "max_tokens": 128, + "max_tokens": ASR_MAX_TOKENS, }) return compare_text( candidate.get("text"), diff --git a/scripts/skippy-tts-oracle.py b/scripts/skippy-tts-oracle.py index 8fa61efbec..3cd8c467c8 100644 --- a/scripts/skippy-tts-oracle.py +++ b/scripts/skippy-tts-oracle.py @@ -160,6 +160,29 @@ def sha256(path: Path) -> str: return digest.hexdigest() +def candidate_test_command(env: dict[str, str]) -> list[str]: + manifest = env.get("SKIPPY_WORKLOAD_PRODUCER_MANIFEST") + if not manifest: + return ["cargo", "test", "--manifest-path", str(ROOT / "Cargo.toml"), + "-p", "skippy-server", "--lib", TEST_NAME, + "--", "--exact", "--nocapture", "--test-threads=1"] + binary_dir = env.get("SKIPPY_WORKLOAD_CANDIDATE_BIN_DIR") + native_dir = env.get("SKIPPY_WORKLOAD_NATIVE_BUILD_DIR") + if not binary_dir or not native_dir: + raise RuntimeError("prebuilt TTS oracle requires workload candidate and native build paths") + # The deterministic waveform probe must use the same source-bound test + # executable as the class smoke, without rebuilding the canary's Metal tree. + subprocess.run( + [sys.executable, str(ROOT / "scripts/check-skippy-workload-candidate.py"), + "--candidate-binary", str(Path(binary_dir) / "skippy-server"), + "--native-build-dir", native_dir, "--producer-manifest", manifest], + cwd=ROOT, env=env, check=True, + ) + producer = json.loads(Path(manifest).read_text(encoding="utf-8")) + return [producer["files"]["test_binary"]["path"], TEST_NAME, + "--exact", "--nocapture", "--test-threads=1"] + + def run_oracle(args: argparse.Namespace) -> dict[str, object]: oracle_cli = Path(args.oracle_cli).resolve() model_path = Path(args.model_path).resolve() @@ -198,9 +221,7 @@ def run_oracle(args: argparse.Namespace) -> dict[str, object]: "SKIPPY_TTS_ORACLE_MAX_FRAMES": str(MAX_FRAMES), }) run_logged( - ["cargo", "test", "--manifest-path", str(ROOT / "Cargo.toml"), - "-p", "skippy-server", "--lib", TEST_NAME, - "--", "--exact", "--nocapture", "--test-threads=1"], + candidate_test_command(candidate_env), work_dir / "tts-candidate-test.log", env=candidate_env, ) diff --git a/scripts/skippy-workload-certify.sh b/scripts/skippy-workload-certify.sh index 30961fd910..a843b1a794 100755 --- a/scripts/skippy-workload-certify.sh +++ b/scripts/skippy-workload-certify.sh @@ -245,6 +245,7 @@ LLAMA_STAGE_BACKEND="$BACKEND" \ "$CANDIDATE_BIN_DIR/skippy-server" serve-openai \ --config "$CONFIG_PATH" \ --bind-addr "127.0.0.1:$PORT" \ + --default-max-tokens 128 \ --telemetry-level off \ >"$SERVER_LOG" 2>&1 & SERVER_PID="$!" diff --git a/scripts/tests/test_skippy_ocr_asr_oracle.py b/scripts/tests/test_skippy_ocr_asr_oracle.py index 3b431b2dee..1c086e3548 100644 --- a/scripts/tests/test_skippy_ocr_asr_oracle.py +++ b/scripts/tests/test_skippy_ocr_asr_oracle.py @@ -79,6 +79,11 @@ def test_asr_matching_refusals_do_not_pass_as_transcripts(self): "I can help you with transcribing audio to text.", None, transcript=True) + def test_asr_repeated_reference_content_is_not_normalized_away(self): + with self.assertRaisesRegex(RuntimeError, "differs from monolithic"): + oracle.compare_text("The mesh is ready", "The mesh is ready. The mesh is ready.", + None, transcript=True) + def test_ocr_sends_same_request_to_both_servers(self): reply = {"choices": [{"message": {"content": "MESH 42"}}]} with patch.object(oracle, "request_json", side_effect=[reply, reply]) as request: @@ -102,8 +107,13 @@ def test_asr_aligns_reference_chat_prompt_with_candidate_audio_route(self): ("http://reference/v1", "/chat/completions")) payload = reference.call_args.args[2] parts = payload["messages"][0]["content"] - self.assertEqual(parts[0]["text"], oracle.ASR_PROMPT) + self.assertEqual(parts[0]["text"], oracle.ASR_PROMPT + "\n") self.assertEqual(parts[1]["input_audio"]["data"], "d2F2") + self.assertEqual(payload["max_tokens"], oracle.ASR_MAX_TOKENS) + # The multipart API has no max_tokens field; set the candidate server's + # default explicitly instead of comparing its CLI default of 16 to 128. + runner = (ROOT / "scripts/skippy-workload-certify.sh").read_text() + self.assertIn(f"--default-max-tokens {oracle.ASR_MAX_TOKENS}", runner) def test_asr_multipart_contains_deterministic_fields_and_audio(self): with patch.object(oracle, "response_json", return_value={"text": "ok"}) as response: diff --git a/scripts/tests/test_skippy_tts_oracle.py b/scripts/tests/test_skippy_tts_oracle.py index 1fdfb53031..19abfe063d 100644 --- a/scripts/tests/test_skippy_tts_oracle.py +++ b/scripts/tests/test_skippy_tts_oracle.py @@ -3,8 +3,10 @@ import argparse from array import array import importlib.util +import json import os from pathlib import Path +import subprocess import sys import tempfile import unittest @@ -32,6 +34,39 @@ def write_wav(path: Path, samples: list[int], *, rate: int = 8000) -> None: class TtsOracleTests(unittest.TestCase): + def test_prebuilt_candidate_requires_verified_producer_and_never_runs_cargo(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + manifest = root / "producer.json" + test_binary = str(root / "candidate tests") + manifest.write_text(json.dumps({"files": {"test_binary": {"path": test_binary}}})) + env = { + "SKIPPY_WORKLOAD_PRODUCER_MANIFEST": str(manifest), + "SKIPPY_WORKLOAD_CANDIDATE_BIN_DIR": str(root / "bin"), + "SKIPPY_WORKLOAD_NATIVE_BUILD_DIR": str(root / "native"), + } + with mock.patch.object(oracle.subprocess, "run") as verify: + command = oracle.candidate_test_command(env) + self.assertEqual([test_binary, oracle.TEST_NAME, "--exact", "--nocapture", "--test-threads=1"], command) + verify.assert_called_once_with( + [sys.executable, str(ROOT / "scripts/check-skippy-workload-candidate.py"), + "--candidate-binary", str(root / "bin/skippy-server"), + "--native-build-dir", str(root / "native"), "--producer-manifest", str(manifest)], + cwd=ROOT, env=env, check=True, + ) + with mock.patch.object(oracle.subprocess, "run", side_effect=subprocess.CalledProcessError(1, ["verify"])): + with self.assertRaises(subprocess.CalledProcessError): + oracle.candidate_test_command(env) + + def test_incomplete_producer_paths_do_not_fall_back_to_cargo(self) -> None: + with self.assertRaisesRegex(RuntimeError, "requires workload candidate and native build paths"): + oracle.candidate_test_command({"SKIPPY_WORKLOAD_PRODUCER_MANIFEST": "producer.json"}) + + def test_standalone_candidate_retains_explicit_cargo_test(self) -> None: + command = oracle.candidate_test_command({}) + self.assertEqual(["cargo", "test"], command[:2]) + self.assertIn(oracle.TEST_NAME, command) + def test_oracle_invocation_matches_candidate_no_repack_context(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: test_root = Path(temp_dir) @@ -55,7 +90,7 @@ def fake_run_logged(command: list[str], _log_path: Path, **_kwargs: object) -> N layer_end=28, work_dir=str(work_dir), ) - with mock.patch.dict(os.environ, {"LLAMA_STAGE_BUILD_DIR": str(test_root / "abi")}): + with mock.patch.dict(os.environ, {"LLAMA_STAGE_BUILD_DIR": str(test_root / "abi")}, clear=True): with mock.patch.object(oracle, "require_pinned_cpu_oracle", return_value="pinned-sha"): with mock.patch.object(oracle, "require_candidate_cpu_static_build"): with mock.patch.object(oracle, "run_logged", side_effect=fake_run_logged): From b8584caaef8ff398df60d2feadee54508c635c30 Mon Sep 17 00:00:00 2001 From: IvGolovach <20299097+IvGolovach@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:03:05 -0700 Subject: [PATCH 08/18] chore(ci): refresh serving console-print source location --- tools/xtask/data/console_print_allowlist.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/xtask/data/console_print_allowlist.json b/tools/xtask/data/console_print_allowlist.json index f4bcf4b8e4..572fb47bd2 100644 --- a/tools/xtask/data/console_print_allowlist.json +++ b/tools/xtask/data/console_print_allowlist.json @@ -4431,7 +4431,7 @@ ], "crates/skippy-server/src/http.rs": [ { - "line": 216, + "line": 223, "macro_name": "println!" } ], From 5934708ea3921aae00833b60e9f2447f0fa1eb86 Mon Sep 17 00:00:00 2001 From: scama Date: Sun, 13 Sep 2026 10:21:47 +1000 Subject: [PATCH 09/18] fix(skippy): satisfy strict PCM conversion lint --- crates/skippy-runtime/src/media.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/skippy-runtime/src/media.rs b/crates/skippy-runtime/src/media.rs index d236804c22..b577dd7d50 100644 --- a/crates/skippy-runtime/src/media.rs +++ b/crates/skippy-runtime/src/media.rs @@ -851,8 +851,10 @@ fn pcm_f32_to_s16le(bytes: &[u8]) -> Result> { return Err(anyhow!("native PCM payload is not aligned to f32 samples")); } let mut output = Vec::with_capacity(bytes.len() / 2); - for sample in bytes.chunks_exact(4) { - let sample = f32::from_ne_bytes(sample.try_into().expect("four-byte PCM sample")); + let (samples, remainder) = bytes.as_chunks::<4>(); + debug_assert!(remainder.is_empty()); + for sample in samples { + let sample = f32::from_ne_bytes(*sample); let quantized = (sample.clamp(-1.0, 1.0) * f32::from(i16::MAX)).round() as i16; output.extend_from_slice(&quantized.to_le_bytes()); } From e84efa7a21288e1effaf4582de821dd42f8038a7 Mon Sep 17 00:00:00 2001 From: scama Date: Sun, 13 Sep 2026 10:59:33 +1000 Subject: [PATCH 10/18] fix(skippy): close non-chat certification gaps --- .../manage-ci/references/current-inventory.md | 8 +- .../src/network/openai/ingress.rs | 32 ++++-- .../src/protocol/convert.rs | 101 ++++++++++++------ crates/openai-frontend/src/audio.rs | 31 +++++- crates/skippy-ffi/build.rs | 7 +- crates/skippy-ffi/src/tests.rs | 7 ++ crates/skippy-runtime/src/media.rs | 23 +++- .../src/frontend/tests/non_chat.rs | 2 + scripts/skippy-family-battery.sh | 16 ++- scripts/skippy-ocr-asr-oracle.py | 6 +- scripts/skippy-workload-certify.sh | 26 +++-- .../test_llama_upstream_canary_contract.py | 20 ++++ scripts/tests/test_skippy_ocr_asr_oracle.py | 4 +- scripts/tests/test_skippy_static_link.py | 18 +++- scripts/tests/test_skippy_workload_certify.py | 11 ++ .../test_verify_workload_oracle_evidence.py | 41 +++++++ scripts/verify-workload-oracle-evidence.py | 2 + scripts/write-workload-oracle-evidence.py | 4 +- ...ippy-reject-exhausted-vocab-sampling.patch | 89 +++++++++++++++ 19 files changed, 379 insertions(+), 69 deletions(-) create mode 100644 third_party/llama.cpp/patches/0027-fix-skippy-reject-exhausted-vocab-sampling.patch diff --git a/.agents/skills/manage-ci/references/current-inventory.md b/.agents/skills/manage-ci/references/current-inventory.md index 44af0c4124..9cbc030f69 100644 --- a/.agents/skills/manage-ci/references/current-inventory.md +++ b/.agents/skills/manage-ci/references/current-inventory.md @@ -48,12 +48,14 @@ runner group (tools come from the runner image; no GitHub Actions model caching). Before native compilation, `scripts/plan-family-battery.py` validates the versioned JSON family policy, the three core parity lanes for certified causal rows, one class-specific -smoke lane for each of the six registry-generated non-chat rows +smoke lane and one local-monolithic oracle lane for each of the six +registry-generated non-chat rows (`embedding`, `rerank`, `encoder_decoder`, `ocr`, `speech_synthesis`, and `speech_recognition`): respectively `embedding-smoke`, `rerank-smoke`, `encoder-decoder-smoke`, `ocr-smoke`, -`speech-synthesis-smoke`, and `speech-recognition-smoke`. These lanes exercise -local full-model and HTTP behavior, without an independent equivalence oracle. It also +`speech-synthesis-smoke`, and `speech-recognition-smoke`, paired with the +corresponding `*-oracle` lane. These certified pairs exercise local full-model +and HTTP behavior and require an independent equivalence oracle. It also checks every exact artifact revision/file in the immutable local cache. It reads only GGUF metadata headers, requires each artifact to have at least one metadata-bearing shard, and requires every shard that carries `*.block_count` and diff --git a/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs b/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs index 9230ab75b8..d5ce822e42 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs @@ -914,6 +914,25 @@ async fn send_media_unsupported( ) } +async fn send_auto_route_rejection( + tcp_stream: ClientStream, + rejection: AutoRouteRejection, + node: &mesh::Node, + request_object_request_ids: &[String], + path: &str, + route_observer: OpenAiRouteObserver<'_>, +) -> proxy::RouteDispatchOutcome { + match rejection { + AutoRouteRejection::MediaUnsupported => { + send_media_unsupported(tcp_stream, route_observer).await + } + AutoRouteRejection::WorkloadUnsupported(workload) => { + proxy::release_request_objects(node, request_object_request_ids).await; + send_workload_unsupported(tcp_stream, workload, path, route_observer).await + } + } +} + fn callable_models_with_local_served( targets: &election::ModelTargets, local_models: Vec, @@ -1143,15 +1162,12 @@ async fn handle_buffered_api_request( let decision = match prepare_auto_route_decision(&mut request, &ctx.route, &descriptors).await { Ok(decision) => decision, - Err(AutoRouteRejection::MediaUnsupported) => { - let outcome = send_media_unsupported(tcp_stream, lifecycle.route_observer()).await; - lifecycle.terminal(terminal_outcome_for_dispatch(outcome)); - return; - } - Err(AutoRouteRejection::WorkloadUnsupported(workload)) => { - let outcome = send_workload_unsupported( + Err(rejection) => { + let outcome = send_auto_route_rejection( tcp_stream, - workload, + rejection, + ctx.route.node, + &request.request_object_request_ids, &request.client_path, lifecycle.route_observer(), ) diff --git a/crates/mesh-llm-host-runtime/src/protocol/convert.rs b/crates/mesh-llm-host-runtime/src/protocol/convert.rs index b1a27b4eed..4f51d39b2a 100644 --- a/crates/mesh-llm-host-runtime/src/protocol/convert.rs +++ b/crates/mesh-llm-host-runtime/src/protocol/convert.rs @@ -329,11 +329,16 @@ fn local_model_metadata_to_proto( fn proto_model_metadata_to_local( metadata: &crate::proto::node::ServedModelMetadata, -) -> crate::mesh::ServedModelMetadata { - crate::mesh::ServedModelMetadata { - workload_class: metadata - .workload_class - .and_then(proto_workload_class_to_local), +) -> Option { + let workload_class = match metadata.workload_class { + Some(value) => { + crate::proto::node::ModelWorkloadClass::try_from(value).ok()?; + proto_workload_class_to_local(value) + } + None => None, + }; + Some(crate::mesh::ServedModelMetadata { + workload_class, architecture: metadata.architecture.clone(), parameter_size: metadata.parameter_size.clone(), parameter_count_b: metadata.parameter_count_b, @@ -346,7 +351,7 @@ fn proto_model_metadata_to_local( kv_head_count: metadata.kv_head_count, expert_count: metadata.expert_count, active_expert_count: metadata.active_expert_count, - } + }) } fn local_workload_class_to_proto(workload: crate::mesh::ModelWorkloadClass) -> i32 { @@ -554,8 +559,8 @@ fn proto_gpu_info_to_legacy_fields(gpus: &[crate::proto::node::GpuInfo]) -> Lega } /// Returns `true` when a proto descriptor carries a non-empty model name. -/// Descriptors without a valid identity are discarded so a partial list -/// cannot suppress the legacy-identity backfill fallback. +/// Descriptors without a valid identity are discarded. A non-empty descriptor +/// list is authoritative, so invalid entries never regain legacy semantics. fn proto_descriptor_has_valid_identity( descriptor: &crate::proto::node::ServedModelDescriptor, ) -> bool { @@ -1050,8 +1055,14 @@ pub(crate) fn proto_ann_to_local( let descriptors: Vec<_> = pa.served_model_descriptors .iter() - .filter(|descriptor| proto_descriptor_has_valid_identity(descriptor)) - .map(|descriptor| { + .filter_map(|descriptor| { + if !proto_descriptor_has_valid_identity(descriptor) { + return None; + } + let metadata = match descriptor.metadata.as_ref() { + Some(metadata) => Some(proto_model_metadata_to_local(metadata)?), + None => None, + }; let capabilities = descriptor .capabilities .as_ref() @@ -1064,7 +1075,7 @@ pub(crate) fn proto_ann_to_local( moe: caps.moe, }) .unwrap_or_default(); - crate::mesh::ServedModelDescriptor { + Some(crate::mesh::ServedModelDescriptor { identity: descriptor .identity .as_ref() @@ -1092,22 +1103,11 @@ pub(crate) fn proto_ann_to_local( }), } }), - metadata: descriptor - .metadata - .as_ref() - .map(proto_model_metadata_to_local), - } + metadata, + }) }) .collect(); - if descriptors.is_empty() { - // All descriptors were invalid — fall back to legacy identity list. - pa.served_model_identities - .iter() - .map(legacy_descriptor_from_identity) - .collect() - } else { - descriptors - } + descriptors } else { pa.served_model_identities .iter() @@ -1399,7 +1399,7 @@ mod tests { }; let proto = local_model_metadata_to_proto(&local); - let restored = proto_model_metadata_to_local(&proto); + let restored = proto_model_metadata_to_local(&proto).expect("known workload class"); assert_eq!(restored.workload_class, Some(workload)); assert_eq!(restored.architecture.as_deref(), Some("test")); @@ -1407,19 +1407,54 @@ mod tests { } #[test] - fn absent_or_unknown_proto_workload_class_is_legacy_compatible() { + fn absent_proto_workload_class_is_legacy_compatible_but_unknown_is_rejected() { let absent = crate::proto::node::ServedModelMetadata::default(); - assert_eq!(proto_model_metadata_to_local(&absent).workload_class, None); + assert_eq!( + proto_model_metadata_to_local(&absent) + .expect("absent workload is legacy-compatible") + .workload_class, + None + ); + let unspecified = crate::proto::node::ServedModelMetadata { + workload_class: Some(crate::proto::node::ModelWorkloadClass::Unspecified as i32), + ..Default::default() + }; + assert_eq!( + proto_model_metadata_to_local(&unspecified) + .expect("explicit unspecified workload is legacy-compatible") + .workload_class, + None + ); let unknown = crate::proto::node::ServedModelMetadata { workload_class: Some(9_999), ..Default::default() }; - assert_eq!( - proto_model_metadata_to_local(&unknown).workload_class, - None, - "newer workload enum values must be ignored by older conversion code" - ); + assert!(proto_model_metadata_to_local(&unknown).is_none()); + } + + #[test] + fn unknown_descriptor_workload_class_does_not_fall_back_to_legacy_identity() { + let identity = crate::proto::node::ServedModelIdentity { + model_name: "future-workload".to_string(), + ..Default::default() + }; + let proto = crate::proto::node::PeerAnnouncement { + endpoint_id: vec![1; 32], + served_model_identities: vec![identity.clone()], + served_model_descriptors: vec![crate::proto::node::ServedModelDescriptor { + identity: Some(identity), + metadata: Some(crate::proto::node::ServedModelMetadata { + workload_class: Some(9_999), + ..Default::default() + }), + ..Default::default() + }], + ..Default::default() + }; + + let (_, announcement) = proto_ann_to_local(&proto).expect("announcement should decode"); + assert!(announcement.served_model_descriptors.is_empty()); } #[test] diff --git a/crates/openai-frontend/src/audio.rs b/crates/openai-frontend/src/audio.rs index 9a54607b57..5dc0f52eb0 100644 --- a/crates/openai-frontend/src/audio.rs +++ b/crates/openai-frontend/src/audio.rs @@ -114,10 +114,10 @@ impl AudioTranscriptionRequest { } if self .temperature - .is_some_and(|value| !value.is_finite() || value < 0.0) + .is_some_and(|value| !value.is_finite() || !(0.0..=1.0).contains(&value)) { return Err(OpenAiError::invalid_request( - "temperature must be a finite non-negative value", + "temperature must be a finite value between 0.0 and 1.0", )); } Ok(()) @@ -128,3 +128,30 @@ impl AudioTranscriptionRequest { pub struct AudioTranscriptionResponse { pub text: String, } + +#[cfg(test)] +mod tests { + use super::*; + + fn transcription(temperature: Option) -> AudioTranscriptionRequest { + AudioTranscriptionRequest { + model: "fixture".to_string(), + file: vec![1], + filename: None, + language: None, + prompt: None, + response_format: "json".to_string(), + temperature, + } + } + + #[test] + fn transcription_temperature_is_bounded_to_openai_range() { + for temperature in [None, Some(0.0), Some(0.5), Some(1.0)] { + assert!(transcription(temperature).validate().is_ok()); + } + for temperature in [Some(-0.1), Some(1.1), Some(f32::NAN), Some(f32::INFINITY)] { + assert!(transcription(temperature).validate().is_err()); + } + } +} diff --git a/crates/skippy-ffi/build.rs b/crates/skippy-ffi/build.rs index 08e9d78f77..ee807bd1d3 100644 --- a/crates/skippy-ffi/build.rs +++ b/crates/skippy-ffi/build.rs @@ -453,7 +453,7 @@ fn cmake_bool_enabled(cache: &std::path::Path, key: &str) -> bool { contents .lines() .find_map(|line| line.strip_prefix(&prefix)) - .is_some_and(|value| matches!(value, "ON" | "TRUE" | "1")) + .is_some_and(|value| matches!(value.trim(), "ON" | "TRUE" | "1")) } fn configured_backend_archive( @@ -465,6 +465,11 @@ fn configured_backend_archive( msvc_archive: &str, ) -> bool { if !selected_backend { + assert!( + !cmake_bool_enabled(cmake_cache, cmake_key), + "unselected backend requires {cmake_key}=OFF in {}", + cmake_cache.display() + ); return false; } assert!( diff --git a/crates/skippy-ffi/src/tests.rs b/crates/skippy-ffi/src/tests.rs index c5b65ac762..dc6cb60914 100644 --- a/crates/skippy-ffi/src/tests.rs +++ b/crates/skippy-ffi/src/tests.rs @@ -36,11 +36,18 @@ fn workload_descriptor_matches_native_layout_and_discriminants() { assert_eq!(offset_of!(WorkloadInfoV1, output_dimensions), 16); assert_eq!(offset_of!(WorkloadInfoV1, classifier_outputs), 20); assert_eq!(offset_of!(WorkloadInfoV1, has_encoder), 24); + assert_eq!(offset_of!(WorkloadInfoV1, has_decoder), 25); + assert_eq!(offset_of!(WorkloadInfoV1, full_model_only), 26); + assert_eq!(offset_of!(WorkloadInfoV1, reserved0), 27); assert_eq!(WorkloadKind::CausalGeneration as i32, 0); assert_eq!(WorkloadKind::Embedding as i32, 1); assert_eq!(WorkloadKind::Rerank as i32, 2); assert_eq!(WorkloadKind::EncoderDecoder as i32, 3); assert_eq!(WorkloadPooling::Unspecified as i32, -1); + assert_eq!(WorkloadPooling::None as i32, 0); + assert_eq!(WorkloadPooling::Mean as i32, 1); + assert_eq!(WorkloadPooling::Cls as i32, 2); + assert_eq!(WorkloadPooling::Last as i32, 3); assert_eq!(WorkloadPooling::Rank as i32, 4); } diff --git a/crates/skippy-runtime/src/media.rs b/crates/skippy-runtime/src/media.rs index b577dd7d50..89c64dda82 100644 --- a/crates/skippy-runtime/src/media.rs +++ b/crates/skippy-runtime/src/media.rs @@ -193,9 +193,16 @@ impl StageModel { free_error(error); } } + struct EmbeddingsGuard(*mut skippy_ffi::Opaque); + impl Drop for EmbeddingsGuard { + fn drop(&mut self) { + unsafe { skippy_ffi::llama_set_embeddings(self.0, false) }; + } + } session.reset()?; unsafe { skippy_ffi::llama_set_embeddings(lctx, true) }; + let _embeddings = EmbeddingsGuard(lctx); let mut guard_error = ptr::null_mut(); let status = unsafe { skippy_ffi::skippy_session_begin_external_decode(session.raw, &mut guard_error) @@ -257,6 +264,7 @@ impl StageModel { return Err(anyhow!("speech backbone did not produce a hidden state")); } let mut generated_frames = 0usize; + let mut stopped = false; while generated_frames < config.max_frames { if cancellation_requested() { return Err(anyhow!("speech synthesis cancelled")); @@ -278,12 +286,19 @@ impl StageModel { )); } if stop || next_hidden_state.is_null() { + stopped = true; break; } generated_frames += 1; hidden_state = next_hidden_state; sampled = session.sample_current(Some(&sampling))?; } + if !stopped { + return Err(anyhow!( + "speech synthesis exceeded the configured {} frame limit", + config.max_frames + )); + } let mut sample_rate = 0_i32; let mut data = ptr::null(); @@ -874,9 +889,11 @@ mod tests { .collect::>(); let converted = pcm_f32_to_s16le(&bytes).expect("aligned native PCM"); - let actual = converted - .chunks_exact(2) - .map(|sample| i16::from_le_bytes(sample.try_into().unwrap())) + let (converted_samples, remainder) = converted.as_chunks::<2>(); + assert!(remainder.is_empty()); + let actual = converted_samples + .iter() + .map(|sample| i16::from_le_bytes(*sample)) .collect::>(); assert_eq!( diff --git a/crates/skippy-server/src/frontend/tests/non_chat.rs b/crates/skippy-server/src/frontend/tests/non_chat.rs index 1327d75c87..512b2207e9 100644 --- a/crates/skippy-server/src/frontend/tests/non_chat.rs +++ b/crates/skippy-server/src/frontend/tests/non_chat.rs @@ -190,6 +190,7 @@ async fn certify_embedding(backend: &StageOpenAiBackend) -> Result<()> { .await?; assert_eq!(first.object, "list"); assert_eq!(first.data.len(), 2); + assert_eq!(first.data.len(), second.data.len()); assert!(first.usage.prompt_tokens > 0); for (left, right) in first.data.iter().zip(&second.data) { let (EmbeddingOutput::Float(left), EmbeddingOutput::Float(right)) = @@ -223,6 +224,7 @@ async fn certify_rerank(backend: &StageOpenAiBackend) -> Result<()> { .await?; let second = backend.rerank(request, OpenAiRequestContext::new()).await?; assert_eq!(first.results.len(), 2); + assert_eq!(first.results.len(), second.results.len()); assert!(first.usage.prompt_tokens > 0); for (left, right) in first.results.iter().zip(&second.results) { assert_eq!(left.index, right.index); diff --git a/scripts/skippy-family-battery.sh b/scripts/skippy-family-battery.sh index 1949c2b658..5eb70d4d80 100755 --- a/scripts/skippy-family-battery.sh +++ b/scripts/skippy-family-battery.sh @@ -4,9 +4,10 @@ set -euo pipefail # Supported-families certification battery (issue #1434; tiers dropped 2026-08-25). # # Causal-generation rows get core split certification: single-step, chain, -# and state-handoff lanes. Non-chat rows get one provisional class-specific -# smoke lane through the Skippy runtime and OpenAI-compatible frontend; -# these classes deliberately fail closed if asked to stage. Models with +# and state-handoff lanes. Non-chat rows get a certified class-specific smoke +# lane through the Skippy runtime and OpenAI-compatible frontend plus an +# independent local-monolithic oracle lane. These classes deliberately fail +# closed if asked to stage. Models with # MTP/NextN tensors require the native draft sideband and verify it against the # target in the correctness lanes. Dense causal rows run them at the first, # midpoint, and last interior cuts. Hybrid/recurrent rows (sweep_period > 0) @@ -897,6 +898,7 @@ run_workload_certify() { --model-path "$target" --model-id "$model_id" --work-dir "$cert_run_dir" + --startup-timeout-secs "$startup_timeout" --skip-build ) if [[ -n "$mmproj" ]]; then @@ -923,11 +925,17 @@ run_workload_certify() { oracle_executable="$SKIPPY_WORKLOAD_ORACLE_TTS" fi if (( certified == 1 )); then + command+=(--require-oracle) + if (( DRY_RUN == 1 )); then + echo "==> workload certification: family=$family class=$model_class lanes=$lane_csv model=$(basename "$target")" + printf '%q ' "${command[@]}" + printf '\n' + return 0 + fi if (( oracle_requested != 1 )); then echo "certified workload $family ($model_class) requires a class-appropriate local-monolithic oracle executable" >&2 exit 1 fi - command+=(--require-oracle) fi echo "==> workload certification: family=$family class=$model_class lanes=$lane_csv model=$(basename "$target")" if (( DRY_RUN == 1 )); then diff --git a/scripts/skippy-ocr-asr-oracle.py b/scripts/skippy-ocr-asr-oracle.py index b205c2e5bc..5f22fa9fc6 100644 --- a/scripts/skippy-ocr-asr-oracle.py +++ b/scripts/skippy-ocr-asr-oracle.py @@ -66,12 +66,12 @@ def compare_text(candidate: object, reference: object, expected: str | None, ) if expected is not None: expected_text = normalized_text(expected, "fixture label") - if expected_text not in candidate_text: + if expected_text != candidate_text: raise RuntimeError( - "output misses independently known fixture text: " + "output does not exactly match independently known fixture text: " f"expected={expected_text!r}, actual={candidate_text!r}" ) - return f"identical normalized text containing {expected_text!r}" + return f"identical normalized text exactly matching {expected_text!r}" return "identical normalized text; unlabeled fixture, no accuracy claim" diff --git a/scripts/skippy-workload-certify.sh b/scripts/skippy-workload-certify.sh index a843b1a794..4194df38f0 100755 --- a/scripts/skippy-workload-certify.sh +++ b/scripts/skippy-workload-certify.sh @@ -13,12 +13,14 @@ ORACLE_SERVER="" ORACLE_COMPLETION="" ORACLE_TTS="" ORACLE_REQUIRED=0 +STARTUP_TIMEOUT_SECS=180 usage() { cat >&2 <<'EOF' usage: scripts/skippy-workload-certify.sh --class CLASS --lane LANE --model-path PATH --model-id ID --work-dir PATH [--projector-path PATH] [--oracle-server PATH] [--oracle-completion PATH] [--oracle-tts PATH] + [--startup-timeout-secs SECONDS] [--require-oracle] # fail closed unless the class-appropriate oracle is selected [--skip-build] # oracle runs require a prebuilt SKIPPY_WORKLOAD_PRODUCER_MANIFEST EOF @@ -35,6 +37,7 @@ while (( $# > 0 )); do --oracle-server) ORACLE_SERVER="$2"; shift ;; --oracle-completion) ORACLE_COMPLETION="$2"; shift ;; --oracle-tts) ORACLE_TTS="$2"; shift ;; + --startup-timeout-secs) STARTUP_TIMEOUT_SECS="$2"; shift ;; --require-oracle) ORACLE_REQUIRED=1 ;; --skip-build) SKIP_BUILD=1 ;; -h|--help) usage; exit 0 ;; @@ -43,6 +46,11 @@ while (( $# > 0 )); do shift done +if [[ ! "$STARTUP_TIMEOUT_SECS" =~ ^[1-9][0-9]*$ ]]; then + echo "--startup-timeout-secs must be a positive integer" >&2 + exit 1 +fi + case "$MODEL_CLASS" in embedding) EXPECTED_LANE="embedding-smoke" ;; rerank) EXPECTED_LANE="rerank-smoke" ;; @@ -262,7 +270,7 @@ cleanup() { } trap cleanup EXIT -for _ in {1..180}; do +for (( attempt = 0; attempt < STARTUP_TIMEOUT_SECS; attempt++ )); do if ! kill -0 "$SERVER_PID" >/dev/null 2>&1; then echo "$MODEL_CLASS OpenAI server exited early" >&2 sed -n '1,240p' "$SERVER_LOG" >&2 @@ -301,7 +309,7 @@ if [[ -n "$ORACLE_SERVER" ]]; then ORACLE_LOG="$WORK_DIR/workload-monolithic-oracle-server.log" "$ORACLE_SERVER" "${ORACLE_ARGS[@]}" >"$ORACLE_LOG" 2>&1 & ORACLE_PID="$!" - for _ in {1..180}; do + for (( attempt = 0; attempt < STARTUP_TIMEOUT_SECS; attempt++ )); do if ! kill -0 "$ORACLE_PID" >/dev/null 2>&1; then echo "$MODEL_CLASS monolithic oracle server exited early" >&2 tail -80 "$ORACLE_LOG" >&2 @@ -376,11 +384,11 @@ fi if [[ "$MODEL_CLASS" == "embedding" ]]; then SDK_PYTHON="${SKIPPY_WORKLOAD_SDK_PYTHON:-python3}" - if "$SDK_PYTHON" -c 'import openai' >/dev/null 2>&1; then - "$SDK_PYTHON" "$ROOT/scripts/ci-openai-embeddings-smoke.py" \ - --base-url "http://127.0.0.1:$PORT/v1" \ - --model "$MODEL_ID" - else - echo "official openai-python SDK smoke skipped: package unavailable to $SDK_PYTHON" >&2 - fi + "$SDK_PYTHON" -c 'import openai' >/dev/null 2>&1 || { + echo "official openai-python SDK smoke requires the openai package in $SDK_PYTHON" >&2 + exit 1 + } + "$SDK_PYTHON" "$ROOT/scripts/ci-openai-embeddings-smoke.py" \ + --base-url "http://127.0.0.1:$PORT/v1" \ + --model "$MODEL_ID" fi diff --git a/scripts/tests/test_llama_upstream_canary_contract.py b/scripts/tests/test_llama_upstream_canary_contract.py index 31e7ed2cbb..4db1cc2ca1 100644 --- a/scripts/tests/test_llama_upstream_canary_contract.py +++ b/scripts/tests/test_llama_upstream_canary_contract.py @@ -604,6 +604,12 @@ def _dry_run( json.dumps(policy) + "\n", encoding="utf-8" ) env = os.environ.copy() + for key in ( + "SKIPPY_WORKLOAD_ORACLE_SERVER", + "SKIPPY_WORKLOAD_ORACLE_COMPLETION", + "SKIPPY_WORKLOAD_ORACLE_TTS", + ): + env.pop(key, None) env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}" return subprocess.run( [ @@ -641,6 +647,20 @@ def test_battery_builds_once_then_skips_build_in_each_lane(self) -> None: ) ) + def test_workload_dry_run_needs_no_oracle_and_forwards_startup_deadline(self) -> None: + model = self._model() + model.update({ + "class": "embedding", + "profile": "workload-oracle", + "evidence": {"fixture": "fixture", "comparison": "fixture"}, + }) + model["execution"]["speculative_policy"] = "disabled" + model["resources"]["startup_timeout_secs"] = 600 + result = self._dry_run("--skip-build", models=[model]) + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn("--startup-timeout-secs 600", result.stdout) + self.assertIn("--require-oracle", result.stdout) + def test_family_battery_has_no_activation_wire_dtype_switches(self) -> None: script = BATTERY.read_text(encoding="utf-8") diff --git a/scripts/tests/test_skippy_ocr_asr_oracle.py b/scripts/tests/test_skippy_ocr_asr_oracle.py index 1c086e3548..005da72659 100644 --- a/scripts/tests/test_skippy_ocr_asr_oracle.py +++ b/scripts/tests/test_skippy_ocr_asr_oracle.py @@ -46,8 +46,10 @@ def test_ocr_requires_both_parity_and_known_text(self): self.assertIn("mesh 42", oracle.compare_text("MESH 42", "Mesh 42.", "MESH 42")) with self.assertRaisesRegex(RuntimeError, "differs from monolithic"): oracle.compare_text("MESH 42", "MESH 43", "MESH 42") - with self.assertRaisesRegex(RuntimeError, "misses independently known"): + with self.assertRaisesRegex(RuntimeError, "does not exactly match independently known"): oracle.compare_text("unrelated", "Unrelated.", "MESH 42") + with self.assertRaisesRegex(RuntimeError, "does not exactly match independently known"): + oracle.compare_text("MESH 42 extra", "Mesh 42 extra.", "MESH 42") def test_asr_unlabeled_fixture_does_not_claim_accuracy(self): detail = oracle.compare_text("The mesh is ready.", "the mesh is ready", None) diff --git a/scripts/tests/test_skippy_static_link.py b/scripts/tests/test_skippy_static_link.py index 91dfb3bf55..1ba17e8ca5 100644 --- a/scripts/tests/test_skippy_static_link.py +++ b/scripts/tests/test_skippy_static_link.py @@ -43,6 +43,11 @@ def setUpClass(cls) -> None: raise RuntimeError(f"build script fixture failed: {result.stderr}") def _run(self, backend: str, flags: dict[str, str]) -> subprocess.CompletedProcess[str]: + return self._run_with_newline(backend, flags, "\n") + + def _run_with_newline( + self, backend: str, flags: dict[str, str], newline: str + ) -> subprocess.CompletedProcess[str]: fixture = tempfile.TemporaryDirectory() self.addCleanup(fixture.cleanup) build_dir = Path(fixture.name) / "native" @@ -51,8 +56,9 @@ def _run(self, backend: str, flags: dict[str, str]) -> subprocess.CompletedProce archive.parent.mkdir(parents=True, exist_ok=True) archive.touch() (build_dir / "CMakeCache.txt").write_text( - "".join(f"{key}:BOOL={value}\n" for key, value in flags.items()), + "".join(f"{key}:BOOL={value}{newline}" for key, value in flags.items()), encoding="utf-8", + newline="", ) env = { key: value for key, value in os.environ.items() @@ -95,6 +101,16 @@ def test_backend_cache_mismatch_fails_closed_despite_stale_archive(self) -> None self.assertNotEqual(0, result.returncode) self.assertIn("selected backend requires GGML_METAL=ON", result.stderr) + def test_unselected_backend_cache_mismatch_fails_closed(self) -> None: + result = self._run("cpu", {"GGML_CUDA": "ON"}) + self.assertNotEqual(0, result.returncode) + self.assertIn("unselected backend requires GGML_CUDA=OFF", result.stderr) + + def test_crlf_cache_values_are_recognized(self) -> None: + result = self._run_with_newline("metal", {"GGML_METAL": "ON"}, "\r\n") + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn("cargo:rustc-link-lib=static=ggml-metal", result.stdout) + if __name__ == "__main__": unittest.main() diff --git a/scripts/tests/test_skippy_workload_certify.py b/scripts/tests/test_skippy_workload_certify.py index 9c3f9d232a..5f98e2e510 100644 --- a/scripts/tests/test_skippy_workload_certify.py +++ b/scripts/tests/test_skippy_workload_certify.py @@ -29,8 +29,19 @@ def test_help_documents_the_typed_certification_inputs(self) -> None: self.assertIn("--oracle-server PATH", result.stderr) self.assertIn("--oracle-completion PATH", result.stderr) self.assertIn("--oracle-tts PATH", result.stderr) + self.assertIn("--startup-timeout-secs SECONDS", result.stderr) self.assertIn("--require-oracle", result.stderr) + def test_startup_timeout_must_be_positive(self) -> None: + result = self._run("--startup-timeout-secs", "0") + self.assertEqual(1, result.returncode) + self.assertIn("must be a positive integer", result.stderr) + + def test_embedding_sdk_smoke_cannot_be_skipped(self) -> None: + runner = RUNNER.read_text(encoding="utf-8") + self.assertIn("official openai-python SDK smoke requires", runner) + self.assertNotIn("SDK smoke skipped", runner) + def test_certified_mode_rejects_missing_oracle_before_build(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: model = Path(temp_dir) / "model.gguf" diff --git a/scripts/tests/test_verify_workload_oracle_evidence.py b/scripts/tests/test_verify_workload_oracle_evidence.py index 7c8a1aebaa..1e9601db6e 100644 --- a/scripts/tests/test_verify_workload_oracle_evidence.py +++ b/scripts/tests/test_verify_workload_oracle_evidence.py @@ -109,6 +109,47 @@ def test_smoke_only_output_cannot_certify_oracle_lane(self) -> None: self.assertEqual(1, result.returncode) self.assertIn("lacks an explicit comparator pass", result.stderr) + def test_writer_rejects_lane_without_smoke_suffix(self) -> None: + comparison_log = Path(self.temp_dir.name) / "comparison.txt" + comparison_log.write_text(self.body["comparison"] + "\n", encoding="utf-8") + result = subprocess.run( + [ + "python3", str(WRITER), "--output", str(self.evidence), + "--comparison-log", str(comparison_log), "--class", "embedding", + "--smoke-lane", "embedding-smoke-extra", "--model-id", "fixture", + "--model-sha256", sha256(self.model), + "--candidate-executable", str(self.candidate), + "--oracle-executable", str(self.oracle), + "--pinned-patch-sha", "a" * 40, "--work-dir", self.temp_dir.name, + ], + cwd=ROOT, text=True, capture_output=True, check=False, + ) + self.assertEqual(1, result.returncode) + self.assertIn("must end with '-smoke'", result.stderr) + + def test_projector_workload_requires_projector_identity(self) -> None: + self.body.update({ + "class": "ocr", + "smoke_lane": "ocr-smoke", + "oracle_lane": "ocr-oracle", + "comparison": "ocr local-monolithic oracle passed: exact text", + }) + self.evidence.write_text(json.dumps(self.body), encoding="utf-8") + result = subprocess.run( + [ + "python3", str(VERIFIER), "--evidence", str(self.evidence), + "--class", "ocr", "--smoke-lane", "ocr-smoke", + "--oracle-lane", "ocr-oracle", "--model-id", "fixture", + "--model-path", str(self.model), + "--candidate-executable", str(self.candidate), + "--oracle-executable", str(self.oracle), + "--pinned-patch-sha", "a" * 40, + ], + cwd=ROOT, text=True, capture_output=True, check=False, + ) + self.assertEqual(1, result.returncode) + self.assertIn("requires a projector path", result.stderr) + if __name__ == "__main__": unittest.main() diff --git a/scripts/verify-workload-oracle-evidence.py b/scripts/verify-workload-oracle-evidence.py index 39beb3b324..8dba7bcb60 100644 --- a/scripts/verify-workload-oracle-evidence.py +++ b/scripts/verify-workload-oracle-evidence.py @@ -29,6 +29,8 @@ def sha256(path: Path) -> str: def verify(args: argparse.Namespace) -> None: + if args.model_class in {"ocr", "speech_synthesis", "speech_recognition"} and args.projector_path is None: + raise ValueError(f"{args.model_class} oracle evidence requires a projector path") evidence = json.loads(args.evidence.read_text(encoding="utf-8")) if not isinstance(evidence, dict): raise ValueError("oracle evidence must be an object") diff --git a/scripts/write-workload-oracle-evidence.py b/scripts/write-workload-oracle-evidence.py index 5d606983eb..4b7b85f7b4 100644 --- a/scripts/write-workload-oracle-evidence.py +++ b/scripts/write-workload-oracle-evidence.py @@ -19,6 +19,8 @@ def sha256(path: Path) -> str: def write_evidence(args: argparse.Namespace) -> None: + if not args.smoke_lane.endswith("-smoke"): + raise ValueError("smoke lane must end with '-smoke'") lines = [ line.strip() for line in args.comparison_log.read_text(encoding="utf-8").splitlines() @@ -32,7 +34,7 @@ def write_evidence(args: argparse.Namespace) -> None: "status": "pass", "class": args.model_class, "smoke_lane": args.smoke_lane, - "oracle_lane": args.smoke_lane.replace("-smoke", "-oracle"), + "oracle_lane": args.smoke_lane.removesuffix("-smoke") + "-oracle", "model_id": args.model_id, "model_sha256": args.model_sha256, "projector_sha256": sha256(args.projector_path) if args.projector_path else None, diff --git a/third_party/llama.cpp/patches/0027-fix-skippy-reject-exhausted-vocab-sampling.patch b/third_party/llama.cpp/patches/0027-fix-skippy-reject-exhausted-vocab-sampling.patch new file mode 100644 index 0000000000..5fa2b7b14e --- /dev/null +++ b/third_party/llama.cpp/patches/0027-fix-skippy-reject-exhausted-vocab-sampling.patch @@ -0,0 +1,89 @@ +From 22034e0315c8a3654849cde0119251ca96c60e46 Mon Sep 17 00:00:00 2001 +From: scama + +Date: Sun, 13 Sep 2026 10:45:49 +1000 +Subject: [PATCH] fix(skippy): reject exhausted vocab sampling + +--- + src/skippy/execution-single.cpp | 6 ++++++ + src/skippy/session.cpp | 5 +++++ + src/skippy/speculative_decoding.cpp | 6 ++++++ + src/skippy/verification.cpp | 6 ++++++ + 4 files changed, 23 insertions(+) + +diff --git a/src/skippy/execution-single.cpp b/src/skippy/execution-single.cpp +index c0153bf77..357222c13 100644 +--- a/src/skippy/execution-single.cpp ++++ b/src/skippy/execution-single.cpp +@@ -282,6 +282,12 @@ enum skippy_status skippy_verify_tokens( + const int32_t n_tokens = static_cast(token_count); + for (int32_t i = 0; i < n_tokens; ++i) { + output_tokens[i] = skippy_greedy_sample_ith(session, i); ++ if (output_tokens[i] == LLAMA_TOKEN_NULL) { ++ *out_token_count = 0; ++ skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, ++ "verification produced no eligible token after vocabulary suppression"); ++ return SKIPPY_STATUS_RUNTIME_ERROR; ++ } + } + } + return status; +diff --git a/src/skippy/session.cpp b/src/skippy/session.cpp +index 45f2e98c0..50af78cb8 100644 +--- a/src/skippy/session.cpp ++++ b/src/skippy/session.cpp +@@ -260,6 +260,11 @@ enum skippy_status skippy_session_sample_current( + return SKIPPY_STATUS_INVALID_ARGUMENT; + } + *out_predicted_token = skippy_sample_token(session, sampling); ++ if (*out_predicted_token == LLAMA_TOKEN_NULL) { ++ skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, ++ "sampling produced no eligible token after vocabulary suppression"); ++ return SKIPPY_STATUS_RUNTIME_ERROR; ++ } + return skippy_success(out_error); + } + +diff --git a/src/skippy/speculative_decoding.cpp b/src/skippy/speculative_decoding.cpp +index 88b3ba0a0..b28ef9096 100644 +--- a/src/skippy/speculative_decoding.cpp ++++ b/src/skippy/speculative_decoding.cpp +@@ -460,6 +460,9 @@ enum skippy_status skippy_mtp_propose_next( + } + + token = skippy_greedy_sample_context(llama_get_model(mtp_ctx), mtp_ctx, -1); ++ if (token == LLAMA_TOKEN_NULL) { ++ break; ++ } + if (out_mtp_draft != nullptr) { + out_mtp_draft->token_ids[token_count] = token; + } +@@ -520,6 +523,9 @@ enum skippy_status skippy_mtp_propose_next( + } + + token = skippy_greedy_sample_context(llama_get_model(mtp_ctx), mtp_ctx, -1); ++ if (token == LLAMA_TOKEN_NULL) { ++ break; ++ } + if (out_mtp_draft != nullptr) { + out_mtp_draft->token_ids[token_count] = token; + } +diff --git a/src/skippy/verification.cpp b/src/skippy/verification.cpp +index b91d87021..c14e7a0a3 100644 +--- a/src/skippy/verification.cpp ++++ b/src/skippy/verification.cpp +@@ -230,6 +230,12 @@ enum skippy_status skippy_verify_tokens_frame_sampled( + skippy_record_tokens(session, &token_ids[i], 1); + const int32_t logits_index = static_cast(i); + output_tokens[i] = skippy_sample_token_ith(session, sampling, logits_index); ++ if (output_tokens[i] == LLAMA_TOKEN_NULL) { ++ *out_token_count = 0; ++ skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, ++ "sampled verification produced no eligible token after vocabulary suppression"); ++ return SKIPPY_STATUS_RUNTIME_ERROR; ++ } + if (stop_after_first_mismatch && i + 1 < token_count && output_tokens[i] != token_ids[i + 1]) { + // Later rows are conditioned on a rejected token and are not + // authoritative predictions. Stop advancing the sampler and +-- +2.54.0 (Apple Git-157) From 6681ca9b5d601410ceb49f445534db68c06b8c14 Mon Sep 17 00:00:00 2001 From: IvGolovach <20299097+IvGolovach@users.noreply.github.com> Date: Sat, 12 Sep 2026 18:56:07 -0700 Subject: [PATCH 11/18] fix(skippy): harden workload admission and certification Close review gaps in request-object cleanup, unknown workload decoding, multipart validation, and native sampling failure handling. Require complete speech and independently verified oracle artifacts, and cover negative paths with regression tests. --- .../manage-ci/references/current-inventory.md | 14 +- ci/ci.md | 8 + .../src/network/openai/ingress.rs | 7 +- .../ingress_tests/request_object_cleanup.rs | 135 ++++++++ .../openai/moa_gateway/workload_admission.rs | 6 + .../moa_gateway/workload_admission/tests.rs | 12 + .../openai/request_parse/audio_multipart.rs | 7 + .../src/network/openai/workload_routing.rs | 12 + .../network/openai/workload_routing/tests.rs | 37 +++ .../src/protocol/convert.rs | 80 +++-- crates/mesh-llm-types/src/mesh/mod.rs | 4 + crates/openai-frontend/README.md | 7 +- crates/openai-frontend/src/audio.rs | 5 + crates/openai-frontend/src/backend.rs | 5 + crates/openai-frontend/src/embeddings.rs | 8 + crates/openai-frontend/src/rerank.rs | 2 + crates/openai-frontend/src/router.rs | 25 +- .../src/router_tests/non_chat.rs | 62 ++++ crates/skippy-ffi/build.rs | 15 +- crates/skippy-runtime/src/media.rs | 34 +- .../src/media/speech_session_tests.rs | 88 +++++ crates/skippy-runtime/src/native.rs | 1 + crates/skippy-runtime/src/session.rs | 3 + crates/skippy-server/src/frontend/backend.rs | 17 +- .../src/frontend/backend/non_chat.rs | 58 ++++ scripts/check-skippy-workload-candidate.py | 8 + scripts/ci-openai-embeddings-smoke.py | 8 +- scripts/ci-workload-monolithic-oracle.py | 7 + scripts/skippy-family-battery.sh | 30 +- scripts/skippy-ocr-asr-oracle.py | 3 + scripts/skippy-tts-oracle.py | 13 +- scripts/skippy-workload-certify.sh | 82 +++-- scripts/tests/test_skippy_ocr_asr_oracle.py | 5 + scripts/tests/test_skippy_static_link.py | 16 +- scripts/tests/test_skippy_workload_certify.py | 23 ++ .../test_verify_workload_oracle_evidence.py | 42 ++- scripts/tests/test_workload_lane_execution.py | 102 ++++++ scripts/verify-workload-oracle-evidence.py | 5 +- scripts/write-workload-oracle-evidence.py | 7 +- ...sampling-across-all-execution-bounda.patch | 303 ++++++++++++++++++ 40 files changed, 1151 insertions(+), 155 deletions(-) create mode 100644 crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/request_object_cleanup.rs create mode 100644 crates/skippy-runtime/src/media/speech_session_tests.rs create mode 100644 scripts/tests/test_workload_lane_execution.py create mode 100644 third_party/llama.cpp/patches/0028-skippy-validate-sampling-across-all-execution-bounda.patch diff --git a/.agents/skills/manage-ci/references/current-inventory.md b/.agents/skills/manage-ci/references/current-inventory.md index 9cbc030f69..5d34fb90f3 100644 --- a/.agents/skills/manage-ci/references/current-inventory.md +++ b/.agents/skills/manage-ci/references/current-inventory.md @@ -47,15 +47,17 @@ default-branch content only on the persistent self-hosted `family-certify` runner group (tools come from the runner image; no GitHub Actions model caching). Before native compilation, `scripts/plan-family-battery.py` validates the versioned JSON family policy, -the three core parity lanes for certified causal rows, one class-specific -smoke lane and one local-monolithic oracle lane for each of the six -registry-generated non-chat rows +the three core parity lanes for certified causal rows, a class-specific +smoke plus independent local-monolithic oracle pair for each of the six registry-generated non-chat rows (`embedding`, `rerank`, `encoder_decoder`, `ocr`, `speech_synthesis`, and `speech_recognition`): respectively `embedding-smoke`, `rerank-smoke`, `encoder-decoder-smoke`, `ocr-smoke`, -`speech-synthesis-smoke`, and `speech-recognition-smoke`, paired with the -corresponding `*-oracle` lane. These certified pairs exercise local full-model -and HTTP behavior and require an independent equivalence oracle. It also +`speech-synthesis-smoke`, and `speech-recognition-smoke`, each paired with its +`-oracle` lane. These lanes exercise local full-model and HTTP behavior and +independent equivalence. Workload readiness uses the planned per-model deadline +for both servers; embedding certification requires the official Python SDK smoke. +Dry-run planning needs no oracle tools; a missing execution prerequisite records +failed lanes and does not discard later family results. It also checks every exact artifact revision/file in the immutable local cache. It reads only GGUF metadata headers, requires each artifact to have at least one metadata-bearing shard, and requires every shard that carries `*.block_count` and diff --git a/ci/ci.md b/ci/ci.md index 937edec111..1a9e0b263b 100644 --- a/ci/ci.md +++ b/ci/ci.md @@ -72,6 +72,14 @@ not rebuild under `--skip-build`. Model/projector cache integrity is still checked before execution, and populating the read-only lab cache remains an external runner operation. +Every certified non-chat row requires both its smoke and independent oracle +lane. Dry runs print these commands without requiring provisioned oracle +binaries; an executable run records missing-oracle failures and continues to +the remaining rows. Each row's planned startup deadline applies separately to +the candidate and reference HTTP servers. Embedding certification also requires +the official local-endpoint Python SDK smoke; a missing SDK is a failure, not +a skipped passing check. + The competitive benchmark can optionally download exact-cohort history from `MESH_PERFORMANCE_HISTORY_DATASET`, validate the checked-in schema, report diff --git a/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs b/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs index d5ce822e42..b9beced35f 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs @@ -317,7 +317,6 @@ async fn resolve_auto_routed_model( return AutoRouteResolution::WorkloadUnsupported(workload); } let Some(available) = router::filter_media_compatible_candidates(&available, &media) else { - proxy::release_request_objects(node, &request.request_object_request_ids).await; return AutoRouteResolution::MediaUnsupported; }; let available = auto_route_pool_for_ready_models( @@ -922,12 +921,12 @@ async fn send_auto_route_rejection( path: &str, route_observer: OpenAiRouteObserver<'_>, ) -> proxy::RouteDispatchOutcome { + proxy::release_request_objects(node, request_object_request_ids).await; match rejection { AutoRouteRejection::MediaUnsupported => { send_media_unsupported(tcp_stream, route_observer).await } AutoRouteRejection::WorkloadUnsupported(workload) => { - proxy::release_request_objects(node, request_object_request_ids).await; send_workload_unsupported(tcp_stream, workload, path, route_observer).await } } @@ -1441,6 +1440,10 @@ mod automatic_routing; #[path = "ingress_tests/audio_workloads.rs"] mod audio_workloads; +#[cfg(test)] +#[path = "ingress_tests/request_object_cleanup.rs"] +mod request_object_cleanup; + #[cfg(test)] #[path = "ingress_tests/tests.rs"] mod tests; diff --git a/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/request_object_cleanup.rs b/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/request_object_cleanup.rs new file mode 100644 index 0000000000..8d0ec806b0 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/request_object_cleanup.rs @@ -0,0 +1,135 @@ +//! Request-object ownership ends exactly once when automatic admission fails. + +use super::*; +use crate::plugin::{BridgeFuture, PluginManager, PluginRpcBridge, RpcResult, proto}; +use std::sync::{Arc, Mutex}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; + +#[derive(Default)] +struct CompletionRecorder(Mutex>); + +impl PluginRpcBridge for CompletionRecorder { + fn handle_request( + &self, + plugin_name: String, + method: String, + params_json: String, + ) -> BridgeFuture> { + assert_eq!(plugin_name, "blobstore"); + assert_eq!(method, "tools/call"); + let params: serde_json::Value = serde_json::from_str(¶ms_json).unwrap(); + assert_eq!( + params["name"], + crate::plugins::blobstore::COMPLETE_REQUEST_TOOL + ); + let request_id = params["arguments"]["request_id"] + .as_str() + .unwrap() + .to_owned(); + self.0.lock().unwrap().push(request_id.clone()); + Box::pin(async move { + let response = rmcp::model::CallToolResult::structured(serde_json::json!({ + "request_id": request_id, "removed_tokens": 1, "removed_bytes": 4, + })); + Ok(RpcResult { + result_json: serde_json::to_string(&response).unwrap(), + }) + }) + } + + fn handle_notification(&self, _: String, _: String, _: String) -> BridgeFuture<()> { + Box::pin(async {}) + } +} + +async fn rejected_request_releases_objects(model: &str, media: bool) { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Worker) + .await + .unwrap(); + let recorder = Arc::new(CompletionRecorder::default()); + let manager = PluginManager::for_test_bridge(&["blobstore"], recorder.clone()); + manager.set_test_capability_providers(vec![crate::plugin::PluginCapabilityProvider { + capability: crate::plugins::blobstore::OBJECT_STORE_CAPABILITY.into(), + plugin_name: "blobstore".into(), + plugin_status: "running".into(), + endpoint_id: None, + available: true, + detail: None, + }]); + node.set_plugin_manager(manager.clone()).await; + let mut targets = election::ModelTargets::default(); + targets.targets.insert( + "text-only".into(), + vec![election::InferenceTarget::Local(1)], + ); + node.set_served_model_descriptors(vec![mesh::ServedModelDescriptor { + identity: mesh::ServedModelIdentity { + model_name: "text-only".into(), + ..Default::default() + }, + capabilities_known: true, + metadata: Some(mesh::ServedModelMetadata { + workload_class: Some(mesh::ModelWorkloadClass::CausalGeneration), + ..Default::default() + }), + ..Default::default() + }]) + .await; + let path = if media { + "/v1/chat/completions" + } else { + "/v1/embeddings" + }; + let body = if media { + serde_json::json!({"model": model, "messages": [{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}} + ]}]}) + } else { + serde_json::json!({"model": model, "input": "hello"}) + } + .to_string(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let handler = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut request = proxy::read_http_request(&mut stream).await.unwrap(); + // Parsing/normalization already owns these objects when this handler starts. + request.request_object_request_ids = vec!["upload-a".into(), "upload-b".into()]; + let affinity = affinity::AffinityRouter::new(); + handle_buffered_api_request( + stream.into(), + request, + ProxyConnectionContext { + route: IngressRouteContext { + node: &node, + targets: &targets, + affinity: &affinity, + plugin_manager: Some(&manager), + }, + }, + None, + crate::runtime::IngressType::LocalOpenAi, + ) + .await; + }); + let mut client = TcpStream::connect(address).await.unwrap(); + client.write_all(format!("POST {path} HTTP/1.1\r\nHost: localhost\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", body.len()).as_bytes()).await.unwrap(); + let mut response = String::new(); + client.read_to_string(&mut response).await.unwrap(); + handler.await.unwrap(); + assert!(response.starts_with("HTTP/1.1 422"), "{response}"); + assert_eq!(*recorder.0.lock().unwrap(), ["upload-a", "upload-b"]); +} + +#[tokio::test] +async fn workload_and_media_rejections_complete_each_request_object_once() { + for (model, media) in [("text-only", false), ("auto", false), ("auto", true)] { + tokio::time::timeout( + std::time::Duration::from_secs(10), + rejected_request_releases_objects(model, media), + ) + .await + .expect("rejected ingress must complete"); + } +} diff --git a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/workload_admission.rs b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/workload_admission.rs index fe1005db75..a3c1469249 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/workload_admission.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/workload_admission.rs @@ -10,6 +10,7 @@ use crate::mesh::{self, ModelWorkloadClass, ServedModelDescriptor}; #[cfg(test)] mod tests; +/// Accept causal chat or legacy metadata, not standalone encoder-decoder support. pub(super) fn descriptor_supports_committee(descriptor: &ServedModelDescriptor) -> bool { matches!( descriptor @@ -20,6 +21,7 @@ pub(super) fn descriptor_supports_committee(descriptor: &ServedModelDescriptor) ) } +/// Resolve model aliases before deciding whether any descriptor admits chat roles. pub(super) fn model_supports_committee(model: &str, descriptors: &[ServedModelDescriptor]) -> bool { let mut matching = descriptors .iter() @@ -46,6 +48,9 @@ pub(super) async fn eligible_targets( InferenceTarget::Local(_) => local.as_slice(), InferenceTarget::Remote(id) => { let Some(peer) = state.peers.get(id) else { + // A committee reserves work on known members. Unlike + // direct legacy routing, a vanished peer only shrinks + // the pool; it must not receive a committee role. return false; }; peer.served_model_descriptors.as_slice() @@ -58,6 +63,7 @@ pub(super) async fn eligible_targets( .collect() } +/// Apply committee admission to each known peer without borrowing another's class. pub(super) async fn eligible_remote_hosts( node: &mesh::Node, model: &str, diff --git a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/workload_admission/tests.rs b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/workload_admission/tests.rs index 51c311192a..03592c2224 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/workload_admission/tests.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/workload_admission/tests.rs @@ -4,6 +4,18 @@ use super::*; use crate::inference::election::ModelTargets; use crate::network::affinity::AffinityRouter; +#[test] +fn explicit_unknown_workload_is_not_a_legacy_committee_member() { + let peer = classified_peer(1, ModelWorkloadClass::Unknown); + assert!(!descriptor_supports_committee( + &peer.served_model_descriptors[0] + )); + assert!(!model_supports_committee( + BIG_MODELS[0].name, + &peer.served_model_descriptors + )); +} + fn classified_peer(seed: u32, class: ModelWorkloadClass) -> mesh::PeerInfo { let mut peer = fleet_peer(seed, BIG_MODELS[0]); peer.served_model_descriptors[0] diff --git a/crates/mesh-llm-host-runtime/src/network/openai/request_parse/audio_multipart.rs b/crates/mesh-llm-host-runtime/src/network/openai/request_parse/audio_multipart.rs index ddd26af4ab..ef32e3fb61 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/request_parse/audio_multipart.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/request_parse/audio_multipart.rs @@ -2,6 +2,7 @@ use anyhow::{Context, Result, bail}; use super::{CRLF, CRLF_HEADER_TERMINATOR, MAX_HEADER_BYTES}; +/// Extract an HTTP form boundary only if it satisfies the bounded ASCII grammar. pub(super) fn multipart_boundary(content_type: &str) -> Option<&str> { let mut parts = content_type.split(';'); if !parts @@ -25,6 +26,7 @@ pub(super) fn multipart_boundary(content_type: &str) -> Option<&str> { valid.then_some(boundary) } +/// Find a nonempty byte marker without decoding the surrounding media payload. fn find_subslice(haystack: &[u8], needle: &[u8]) -> Option { (!needle.is_empty()) .then(|| { @@ -35,6 +37,7 @@ fn find_subslice(haystack: &[u8], needle: &[u8]) -> Option { .flatten() } +/// Ignore marker-like file bytes unless followed by a valid delimiter suffix. fn find_multipart_boundary(body: &[u8], marker: &[u8], from: usize) -> Option { let mut cursor = from; while let Some(offset) = find_subslice(&body[cursor..], marker) { @@ -48,6 +51,7 @@ fn find_multipart_boundary(body: &[u8], marker: &[u8], from: usize) -> Option Result> { let mut parameters = Vec::new(); let mut start = 0; @@ -75,6 +79,7 @@ fn disposition_parameters(value: &str) -> Result> { Ok(parameters) } +/// Identify the model field without mistaking quoted filename text for its name. pub(super) fn multipart_part_is_model(headers: &str) -> Result { let mut disposition = None; for line in headers.split("\r\n") { @@ -120,6 +125,7 @@ pub(super) fn multipart_part_is_model(headers: &str) -> Result { Ok(field_name == Some("model")) } +/// Locate the unique model field while validating framing and bounded part headers. pub(super) fn multipart_model_value_range( content_type: &str, body: &[u8], @@ -171,6 +177,7 @@ pub(super) fn multipart_model_value_range( } } +/// Read the routed model as bounded UTF-8, leaving all uploaded file bytes intact. pub(super) fn multipart_model_field(content_type: &str, body: &[u8]) -> Result> { let Some(range) = multipart_model_value_range(content_type, body)? else { return Ok(None); diff --git a/crates/mesh-llm-host-runtime/src/network/openai/workload_routing.rs b/crates/mesh-llm-host-runtime/src/network/openai/workload_routing.rs index 1b3e70be55..00f180d7ca 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/workload_routing.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/workload_routing.rs @@ -12,6 +12,7 @@ use crate::mesh::{self, ModelWorkloadClass, ServedModelDescriptor}; #[cfg(test)] mod tests; +/// Recognize binary audio-to-text endpoints independently of query parameters. pub(super) fn is_audio_upload_path(path: &str) -> bool { matches!( path.split('?').next().unwrap_or(path), @@ -19,6 +20,7 @@ pub(super) fn is_audio_upload_path(path: &str) -> bool { ) } +/// Map inference endpoints to native workloads; control paths impose no class. pub(super) fn request_workload_class(path: &str) -> Option { match path.split('?').next().unwrap_or(path) { "/v1/chat/completions" @@ -42,6 +44,7 @@ pub(super) fn supports_generation_affinity(path: &str) -> bool { ) } +/// Expose identity hints only for endpoints whose generation state is reusable. pub(super) fn affinity_body( request: &super::request_parse::BufferedHttpRequest, ) -> Option<&serde_json::Value> { @@ -74,11 +77,13 @@ pub(super) async fn ingress_candidates( eligible_targets(node, model, path, &remote).await } +/// Preserve absent legacy chat metadata, but never authorize an unknown class. fn class_is_compatible( requested: ModelWorkloadClass, advertised: Option, ) -> bool { match (requested, advertised) { + (ModelWorkloadClass::Unknown, _) | (_, Some(ModelWorkloadClass::Unknown)) => false, (ModelWorkloadClass::CausalGeneration, None) => true, ( ModelWorkloadClass::CausalGeneration, @@ -89,6 +94,7 @@ fn class_is_compatible( } } +/// Check discoverability across matching descriptors, not target eligibility. pub(super) fn model_satisfies_workload_class( model: &str, requested: ModelWorkloadClass, @@ -112,6 +118,7 @@ pub(super) fn model_satisfies_workload_class( }) } +/// Require verified audio and a known generation class on the same descriptor. fn descriptor_supports_audio_upload(descriptor: &ServedModelDescriptor) -> bool { descriptor.capabilities_known && descriptor.capabilities.supports_audio_runtime() @@ -124,6 +131,7 @@ fn descriptor_supports_audio_upload(descriptor: &ServedModelDescriptor) -> bool ) } +/// Apply endpoint-specific admission, including stricter audio upload metadata. pub(super) fn model_satisfies_request_workload( model: &str, workload: ModelWorkloadClass, @@ -140,6 +148,7 @@ pub(super) fn model_satisfies_request_workload( } } +/// Select one matching descriptor that independently supports the endpoint. pub(super) fn descriptor_for_request<'a>( model: &str, path: &str, @@ -158,6 +167,7 @@ pub(super) fn descriptor_for_request<'a>( }) } +/// Infer binary upload requirements without interpreting file bytes as JSON. pub(super) fn request_media( path: &str, body: Option<&serde_json::Value>, @@ -212,6 +222,7 @@ pub(super) fn routing_candidates<'a>( .collect() } +/// Filter each candidate against its own advertisement before routing decisions. pub(super) async fn eligible_targets( node: &mesh::Node, model: &str, @@ -240,6 +251,7 @@ pub(super) async fn eligible_targets( .collect() } +/// Restrict peer IDs using the same per-target workload policy as local ingress. pub(super) async fn eligible_remote_hosts( node: &mesh::Node, model: &str, diff --git a/crates/mesh-llm-host-runtime/src/network/openai/workload_routing/tests.rs b/crates/mesh-llm-host-runtime/src/network/openai/workload_routing/tests.rs index 8cb3ff1fad..135ea5822d 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/workload_routing/tests.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/workload_routing/tests.rs @@ -1,5 +1,42 @@ use super::*; +#[test] +fn unknown_workloads_never_inherit_legacy_admission() { + let metadata: mesh::ServedModelMetadata = serde_json::from_value(serde_json::json!({ + "workload_class": "future_non_chat_contract" + })) + .unwrap(); + assert_eq!(metadata.workload_class, Some(ModelWorkloadClass::Unknown)); + let descriptors = [mesh::ServedModelDescriptor { + metadata: Some(metadata), + ..local_gguf_descriptor("future") + }]; + for path in [ + "/v1/chat/completions", + "/v1/completions", + "/v1/responses", + "/v1/embeddings", + "/v1/rerank", + "/v1/audio/speech", + "/v1/audio/transcriptions", + "/v1/audio/translations", + ] { + assert!( + !model_satisfies_request_workload( + "future", + request_workload_class(path).unwrap(), + path, + &descriptors + ), + "{path}" + ); + } + let relayed = serde_json::to_value(&descriptors[0].metadata).unwrap(); + assert_eq!(relayed["workload_class"], "unknown"); + let absent: mesh::ServedModelMetadata = serde_json::from_value(serde_json::json!({})).unwrap(); + assert_eq!(absent.workload_class, None); +} + fn local_gguf_descriptor(model_name: &str) -> ServedModelDescriptor { ServedModelDescriptor { identity: mesh::ServedModelIdentity { diff --git a/crates/mesh-llm-host-runtime/src/protocol/convert.rs b/crates/mesh-llm-host-runtime/src/protocol/convert.rs index 4f51d39b2a..d0f9b5631a 100644 --- a/crates/mesh-llm-host-runtime/src/protocol/convert.rs +++ b/crates/mesh-llm-host-runtime/src/protocol/convert.rs @@ -329,16 +329,13 @@ fn local_model_metadata_to_proto( fn proto_model_metadata_to_local( metadata: &crate::proto::node::ServedModelMetadata, -) -> Option { - let workload_class = match metadata.workload_class { - Some(value) => { - crate::proto::node::ModelWorkloadClass::try_from(value).ok()?; - proto_workload_class_to_local(value) - } - None => None, - }; - Some(crate::mesh::ServedModelMetadata { - workload_class, +) -> crate::mesh::ServedModelMetadata { + crate::mesh::ServedModelMetadata { + // Keep an explicit unknown descriptor: dropping it can re-enable the + // legacy model-name fallback elsewhere in routing and gossip. + workload_class: metadata + .workload_class + .and_then(proto_workload_class_to_local), architecture: metadata.architecture.clone(), parameter_size: metadata.parameter_size.clone(), parameter_count_b: metadata.parameter_count_b, @@ -351,7 +348,7 @@ fn proto_model_metadata_to_local( kv_head_count: metadata.kv_head_count, expert_count: metadata.expert_count, active_expert_count: metadata.active_expert_count, - }) + } } fn local_workload_class_to_proto(workload: crate::mesh::ModelWorkloadClass) -> i32 { @@ -364,6 +361,9 @@ fn local_workload_class_to_proto(workload: crate::mesh::ModelWorkloadClass) -> i Local::Rerank => Proto::Rerank as i32, Local::EncoderDecoder => Proto::EncoderDecoder as i32, Local::SpeechSynthesis => Proto::SpeechSynthesis as i32, + // Preserve explicit denial when relaying metadata. Zero is the legacy + // unspecified value and must not erase an unknown workload. + Local::Unknown => -1, } } @@ -371,13 +371,14 @@ fn proto_workload_class_to_local(value: i32) -> Option None, - Proto::CausalGeneration => Some(Local::CausalGeneration), - Proto::Embedding => Some(Local::Embedding), - Proto::Rerank => Some(Local::Rerank), - Proto::EncoderDecoder => Some(Local::EncoderDecoder), - Proto::SpeechSynthesis => Some(Local::SpeechSynthesis), + match Proto::try_from(value) { + Ok(Proto::CausalGeneration) => Some(Local::CausalGeneration), + Ok(Proto::Embedding) => Some(Local::Embedding), + Ok(Proto::Rerank) => Some(Local::Rerank), + Ok(Proto::EncoderDecoder) => Some(Local::EncoderDecoder), + Ok(Proto::SpeechSynthesis) => Some(Local::SpeechSynthesis), + Ok(Proto::Unspecified) => None, + Err(_) => Some(Local::Unknown), } } @@ -1059,10 +1060,10 @@ pub(crate) fn proto_ann_to_local( if !proto_descriptor_has_valid_identity(descriptor) { return None; } - let metadata = match descriptor.metadata.as_ref() { - Some(metadata) => Some(proto_model_metadata_to_local(metadata)?), - None => None, - }; + let metadata = descriptor + .metadata + .as_ref() + .map(proto_model_metadata_to_local); let capabilities = descriptor .capabilities .as_ref() @@ -1391,6 +1392,7 @@ mod tests { crate::mesh::ModelWorkloadClass::Rerank, crate::mesh::ModelWorkloadClass::EncoderDecoder, crate::mesh::ModelWorkloadClass::SpeechSynthesis, + crate::mesh::ModelWorkloadClass::Unknown, ] { let local = crate::mesh::ServedModelMetadata { workload_class: Some(workload), @@ -1399,7 +1401,7 @@ mod tests { }; let proto = local_model_metadata_to_proto(&local); - let restored = proto_model_metadata_to_local(&proto).expect("known workload class"); + let restored = proto_model_metadata_to_local(&proto); assert_eq!(restored.workload_class, Some(workload)); assert_eq!(restored.architecture.as_deref(), Some("test")); @@ -1409,20 +1411,13 @@ mod tests { #[test] fn absent_proto_workload_class_is_legacy_compatible_but_unknown_is_rejected() { let absent = crate::proto::node::ServedModelMetadata::default(); - assert_eq!( - proto_model_metadata_to_local(&absent) - .expect("absent workload is legacy-compatible") - .workload_class, - None - ); + assert_eq!(proto_model_metadata_to_local(&absent).workload_class, None); let unspecified = crate::proto::node::ServedModelMetadata { workload_class: Some(crate::proto::node::ModelWorkloadClass::Unspecified as i32), ..Default::default() }; assert_eq!( - proto_model_metadata_to_local(&unspecified) - .expect("explicit unspecified workload is legacy-compatible") - .workload_class, + proto_model_metadata_to_local(&unspecified).workload_class, None ); @@ -1430,7 +1425,16 @@ mod tests { workload_class: Some(9_999), ..Default::default() }; - assert!(proto_model_metadata_to_local(&unknown).is_none()); + let local = proto_model_metadata_to_local(&unknown); + assert_eq!( + local.workload_class, + Some(crate::mesh::ModelWorkloadClass::Unknown) + ); + let relayed = local_model_metadata_to_proto(&local); + assert_eq!( + proto_model_metadata_to_local(&relayed).workload_class, + local.workload_class + ); } #[test] @@ -1454,7 +1458,15 @@ mod tests { }; let (_, announcement) = proto_ann_to_local(&proto).expect("announcement should decode"); - assert!(announcement.served_model_descriptors.is_empty()); + assert_eq!(announcement.served_model_descriptors.len(), 1); + assert_eq!( + announcement.served_model_descriptors[0] + .metadata + .as_ref() + .unwrap() + .workload_class, + Some(crate::mesh::ModelWorkloadClass::Unknown) + ); } #[test] diff --git a/crates/mesh-llm-types/src/mesh/mod.rs b/crates/mesh-llm-types/src/mesh/mod.rs index 869fa3c403..1b8ecc1926 100644 --- a/crates/mesh-llm-types/src/mesh/mod.rs +++ b/crates/mesh-llm-types/src/mesh/mod.rs @@ -63,6 +63,10 @@ pub enum ModelWorkloadClass { Rerank, EncoderDecoder, SpeechSynthesis, + /// Explicit metadata from a newer peer that this node cannot interpret. + /// Unlike absent legacy metadata, this never authorizes inference. + #[serde(other)] + Unknown, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] diff --git a/crates/openai-frontend/README.md b/crates/openai-frontend/README.md index a0d5eb27dd..855c22401c 100644 --- a/crates/openai-frontend/README.md +++ b/crates/openai-frontend/README.md @@ -75,7 +75,10 @@ flowchart TB R --> C ``` -The backend boundary is intentionally small: +The backend boundary below is a partial generation example. The complete +[`OpenAiBackend` trait](src/backend.rs) also defines `embeddings`, `rerank`, +`audio_speech`, `audio_transcription`, and `audio_translation`; override their +default unsupported responses to serve the corresponding non-chat endpoints. ```rust #[async_trait] @@ -84,6 +87,7 @@ pub trait OpenAiBackend { async fn chat_completion( &self, request: ChatCompletionRequest, + context: OpenAiRequestContext, ) -> OpenAiResult; async fn chat_completion_stream( &self, @@ -93,6 +97,7 @@ pub trait OpenAiBackend { async fn completion( &self, request: CompletionRequest, + context: OpenAiRequestContext, ) -> OpenAiResult; async fn completion_stream( &self, diff --git a/crates/openai-frontend/src/audio.rs b/crates/openai-frontend/src/audio.rs index 5dc0f52eb0..3358139eb7 100644 --- a/crates/openai-frontend/src/audio.rs +++ b/crates/openai-frontend/src/audio.rs @@ -17,6 +17,7 @@ pub enum AudioFormat { } impl AudioFormat { + /// Return the media type corresponding to the requested audio wire encoding. pub const fn content_type(self) -> &'static str { match self { Self::Mp3 => "audio/mpeg", @@ -40,11 +41,13 @@ pub struct AudioSpeechRequest { pub speed: f32, } +/// Omitted speed retains the model's unscaled playback rate. fn default_speed() -> f32 { 1.0 } impl AudioSpeechRequest { + /// Validate required inputs and the finite OpenAI playback-speed range. pub fn validate(&self) -> OpenAiResult<()> { if self.model.trim().is_empty() || self.input.is_empty() || self.voice.trim().is_empty() { return Err(OpenAiError::invalid_request( @@ -67,6 +70,7 @@ pub struct AudioResponse { } impl AudioResponse { + /// Reject empty backend output before constructing a binary HTTP response. pub fn new(bytes: Vec, content_type: impl Into) -> OpenAiResult { if bytes.is_empty() { return Err(OpenAiError::backend( @@ -94,6 +98,7 @@ pub struct AudioTranscriptionRequest { impl AudioTranscriptionRequest { pub const MAX_FILE_BYTES: usize = MAX_AUDIO_BYTES; + /// Enforce upload size, output format, and the finite `[0, 1]` temperature range. pub fn validate(&self) -> OpenAiResult<()> { if self.model.trim().is_empty() { return Err(OpenAiError::invalid_request("model must not be empty")); diff --git a/crates/openai-frontend/src/backend.rs b/crates/openai-frontend/src/backend.rs index 61d41234ad..f70e470ff9 100644 --- a/crates/openai-frontend/src/backend.rs +++ b/crates/openai-frontend/src/backend.rs @@ -190,6 +190,7 @@ pub trait OpenAiBackend: Send + Sync + 'static { )) } + /// Embed text or token batches. The default rejects unsupported backends. async fn embeddings( &self, _request: EmbeddingsRequest, @@ -200,6 +201,7 @@ pub trait OpenAiBackend: Send + Sync + 'static { )) } + /// Rank documents against a query, preserving their original indexes. async fn rerank( &self, _request: RerankRequest, @@ -210,6 +212,7 @@ pub trait OpenAiBackend: Send + Sync + 'static { )) } + /// Synthesize binary audio; unsupported codecs and voices must return errors. async fn audio_speech( &self, _request: AudioSpeechRequest, @@ -220,6 +223,7 @@ pub trait OpenAiBackend: Send + Sync + 'static { )) } + /// Transcribe uploaded audio in its source language using the request context. async fn audio_transcription( &self, _request: AudioTranscriptionRequest, @@ -230,6 +234,7 @@ pub trait OpenAiBackend: Send + Sync + 'static { )) } + /// Translate uploaded audio into English; unsupported backends fail explicitly. async fn audio_translation( &self, _request: AudioTranscriptionRequest, diff --git a/crates/openai-frontend/src/embeddings.rs b/crates/openai-frontend/src/embeddings.rs index b8c4f33096..96a900f2d9 100644 --- a/crates/openai-frontend/src/embeddings.rs +++ b/crates/openai-frontend/src/embeddings.rs @@ -17,6 +17,7 @@ pub enum EmbeddingInput { } impl EmbeddingInput { + /// Count independent inputs; a single token array is one input, not a batch. pub fn len(&self) -> usize { match self { Self::Text(_) | Self::Tokens(_) => 1, @@ -25,6 +26,7 @@ impl EmbeddingInput { } } + /// Reject empty batches and empty members in text or pre-tokenized input. pub fn is_empty(&self) -> bool { match self { Self::Text(value) => value.is_empty(), @@ -34,6 +36,7 @@ impl EmbeddingInput { } } + /// Detect negative token IDs without applying tokenizer rules to text input. fn contains_invalid_token(&self) -> bool { match self { Self::Text(_) | Self::Texts(_) => false, @@ -55,11 +58,13 @@ pub struct EmbeddingsRequest { pub user: Option, } +/// Preserve the endpoint's default numeric-vector response representation. fn default_encoding_format() -> String { DEFAULT_ENCODING_FORMAT.to_string() } impl EmbeddingsRequest { + /// Validate batch content, output encoding, and optional positive dimensions. pub fn validate(&self) -> OpenAiResult<()> { if self.model.trim().is_empty() { return Err(OpenAiError::invalid_request("model must not be empty")); @@ -117,6 +122,8 @@ pub struct EmbeddingResponse { } impl EmbeddingResponse { + /// Preserve input indexes and encode vectors with the already-validated format. + /// Embedding usage contains prompt tokens only; this endpoint generates none. pub fn from_embeddings( model: String, embeddings: Vec, @@ -149,6 +156,7 @@ impl EmbeddingResponse { } } +/// Encode little-endian IEEE-754 samples, independent of the host byte order. fn encode_f32_base64(values: &[f32]) -> String { let mut bytes = Vec::with_capacity(std::mem::size_of_val(values)); for value in values { diff --git a/crates/openai-frontend/src/rerank.rs b/crates/openai-frontend/src/rerank.rs index b859493ad1..0a77718725 100644 --- a/crates/openai-frontend/src/rerank.rs +++ b/crates/openai-frontend/src/rerank.rs @@ -10,6 +10,7 @@ pub enum RerankDocument { } impl RerankDocument { + /// Borrow plain text or a document object's required string `text` field. pub fn text(&self) -> OpenAiResult<&str> { match self { Self::Text(text) => Ok(text), @@ -37,6 +38,7 @@ pub struct RerankRequest { } impl RerankRequest { + /// Validate every document before admission, including optional top-N bounds. pub fn validate(&self) -> OpenAiResult<()> { if self.model.trim().is_empty() { return Err(OpenAiError::invalid_request("model must not be empty")); diff --git a/crates/openai-frontend/src/router.rs b/crates/openai-frontend/src/router.rs index b278385ee9..61f5d23791 100644 --- a/crates/openai-frontend/src/router.rs +++ b/crates/openai-frontend/src/router.rs @@ -336,6 +336,7 @@ async fn models( })) } +/// Validate embedding input and preserve cancellation, usage, and lifecycle identity. async fn embeddings( State(state): State, Extension(context): Extension, @@ -363,6 +364,7 @@ async fn embeddings( Ok(json_response_with_usage(response, &usage)) } +/// Validate query/documents before invoking the context-bound rerank backend. async fn rerank( State(state): State, Extension(context): Extension, @@ -386,6 +388,7 @@ async fn rerank( Ok(json_response_with_usage(response, &usage)) } +/// Dispatch validated speech input and return binary audio without JSON wrapping. async fn audio_speech( State(state): State, Extension(context): Extension, @@ -407,6 +410,7 @@ async fn audio_speech( audio_response(response) } +/// Validate the backend media type before placing audio bytes in the response. fn audio_response(audio: AudioResponse) -> Result { let content_type = HeaderValue::from_str(&audio.content_type) .map_err(|_| OpenAiError::backend("audio backend returned an invalid content type"))?; @@ -417,6 +421,7 @@ fn audio_response(audio: AudioResponse) -> Result { Ok(response) } +/// Handle source-language transcription through the shared multipart path. async fn audio_transcriptions( State(state): State, Extension(context): Extension, @@ -425,6 +430,7 @@ async fn audio_transcriptions( audio_text_request(state, context, multipart_payload(multipart)?, false).await } +/// Select English translation without changing the multipart upload contract. async fn audio_translations( State(state): State, Extension(context): Extension, @@ -433,12 +439,14 @@ async fn audio_translations( audio_text_request(state, context, multipart_payload(multipart)?, true).await } +/// Convert extractor rejection into the frontend's structured invalid-request error. fn multipart_payload(multipart: Result) -> OpenAiResult { multipart.map_err(|error| { OpenAiError::invalid_request(format!("invalid multipart request: {error}")) }) } +/// Preserve payload-too-large status when a bounded multipart field cannot be read. fn multipart_error(error: MultipartError, field: &str) -> OpenAiError { if error.status() == StatusCode::PAYLOAD_TOO_LARGE { OpenAiError::payload_too_large(format!("{field} is too large: {error}")) @@ -447,6 +455,7 @@ fn multipart_error(error: MultipartError, field: &str) -> OpenAiError { } } +/// Share upload validation and cancellation while retaining endpoint-specific dispatch. async fn audio_text_request( state: FrontendState, context: OpenAiLifecycleContext, @@ -495,6 +504,7 @@ async fn audio_text_request( } } +/// Decode a bounded audio upload, rejecting duplicate recognized fields consistently. async fn parse_audio_multipart( mut multipart: Multipart, ) -> OpenAiResult { @@ -505,6 +515,7 @@ async fn parse_audio_multipart( let mut prompt = None; let mut response_format = None; let mut temperature = None; + let mut seen_fields = std::collections::HashSet::new(); while let Some(field) = multipart .next_field() @@ -512,6 +523,15 @@ async fn parse_audio_multipart( .map_err(|error| multipart_error(error, "multipart body"))? { let name = field.name().unwrap_or_default().to_string(); + if matches!( + name.as_str(), + "model" | "file" | "language" | "prompt" | "response_format" | "temperature" + ) && !seen_fields.insert(name.clone()) + { + return Err(OpenAiError::invalid_request(format!( + "duplicate multipart {name} field" + ))); + } if name == "file" { filename = field.file_name().map(str::to_owned); let bytes = field @@ -521,11 +541,6 @@ async fn parse_audio_multipart( file = Some(bytes.to_vec()); continue; } - if name == "model" && model.is_some() { - return Err(OpenAiError::invalid_request( - "duplicate multipart model field", - )); - } let value = field .text() .await diff --git a/crates/openai-frontend/src/router_tests/non_chat.rs b/crates/openai-frontend/src/router_tests/non_chat.rs index 08766e77e7..1a8abf69dd 100644 --- a/crates/openai-frontend/src/router_tests/non_chat.rs +++ b/crates/openai-frontend/src/router_tests/non_chat.rs @@ -1,5 +1,67 @@ use super::*; +#[tokio::test] +async fn audio_upload_rejects_each_duplicate_field() { + let boundary = "duplicate-audio-field"; + for path in ["/v1/audio/transcriptions", "/v1/audio/translations"] { + for (field, value) in [ + ("model", "audio-model"), + ("file", "WAVE"), + ("language", "en"), + ("prompt", "words"), + ("response_format", "json"), + ("temperature", "0"), + ] { + let part = format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"{field}\"\r\n\r\n{value}\r\n" + ); + let mut body = part.repeat(2).into_bytes(); + body.extend_from_slice(&audio_multipart(boundary, "json")); + let response = post_audio_multipart(path, boundary, body).await; + assert_eq!( + response.status(), + StatusCode::BAD_REQUEST, + "{path}: {field}" + ); + let body = response_body_json(response).await; + assert_eq!( + body["error"]["message"], + format!("duplicate multipart {field} field") + ); + } + } +} + +#[tokio::test] +async fn audio_upload_enforces_temperature_range_at_both_endpoints() { + let boundary = "audio-temperature"; + for path in ["/v1/audio/transcriptions", "/v1/audio/translations"] { + for (value, accepted) in [ + ("0", true), + ("1", true), + ("0.5", true), + ("1.1", false), + ("-0.1", false), + ("NaN", false), + ("inf", false), + ("-inf", false), + ] { + let mut body = format!("--{boundary}\r\nContent-Disposition: form-data; name=\"temperature\"\r\n\r\n{value}\r\n").into_bytes(); + body.extend_from_slice(&audio_multipart(boundary, "json")); + let response = post_audio_multipart(path, boundary, body).await; + assert_eq!( + response.status(), + if accepted { + StatusCode::OK + } else { + StatusCode::BAD_REQUEST + }, + "{path}: {value}" + ); + } + } +} + #[tokio::test] async fn embeddings_route_preserves_batch_order_and_usage() { let response = post_json( diff --git a/crates/skippy-ffi/build.rs b/crates/skippy-ffi/build.rs index ee807bd1d3..1861cf2859 100644 --- a/crates/skippy-ffi/build.rs +++ b/crates/skippy-ffi/build.rs @@ -449,11 +449,7 @@ fn cmake_bool_enabled(cache: &std::path::Path, key: &str) -> bool { let Ok(contents) = std::fs::read_to_string(cache) else { return false; }; - let prefix = format!("{key}:BOOL="); - contents - .lines() - .find_map(|line| line.strip_prefix(&prefix)) - .is_some_and(|value| matches!(value.trim(), "ON" | "TRUE" | "1")) + cmake_cache_bool(&contents, key) } fn configured_backend_archive( @@ -464,16 +460,17 @@ fn configured_backend_archive( unix_archive: &str, msvc_archive: &str, ) -> bool { + let configured = cmake_bool_enabled(cmake_cache, cmake_key); if !selected_backend { assert!( - !cmake_bool_enabled(cmake_cache, cmake_key), - "unselected backend requires {cmake_key}=OFF in {}", + !configured, + "staged backend mismatch: {cmake_key}=ON in {} but LLAMA_STAGE_BACKEND does not select it", cmake_cache.display() ); return false; } assert!( - cmake_bool_enabled(cmake_cache, cmake_key), + configured, "selected backend requires {cmake_key}=ON in {}", cmake_cache.display() ); @@ -739,6 +736,6 @@ fn cmake_cache_value(cache: &str, key: &str) -> Option { fn cmake_cache_bool(cache: &str, key: &str) -> bool { cmake_cache_value(cache, key) - .map(|value| matches!(value.as_str(), "ON" | "TRUE" | "1")) + .map(|value| matches!(value.trim(), "ON" | "TRUE" | "1")) .unwrap_or(false) } diff --git a/crates/skippy-runtime/src/media.rs b/crates/skippy-runtime/src/media.rs index 89c64dda82..12fad95ffd 100644 --- a/crates/skippy-runtime/src/media.rs +++ b/crates/skippy-runtime/src/media.rs @@ -13,12 +13,14 @@ use crate::{ MediaPrefillFrame, SamplingConfig, }; +/// Audio encoding returned by the native speech generator. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SpeechOutputFormat { Wav, PcmS16Le, } +/// Full-model speech inputs and deterministic sampling controls. #[derive(Debug, Clone, PartialEq)] pub struct SpeechSynthesisConfig { pub prompt: String, @@ -30,6 +32,8 @@ pub struct SpeechSynthesisConfig { pub max_frames: usize, } +/// Complete generated audio and its native frame count. Reaching the configured +/// frame limit returns an error instead of this response. #[derive(Debug, Clone, PartialEq, Eq)] pub struct SpeechAudio { pub bytes: Vec, @@ -38,6 +42,18 @@ pub struct SpeechAudio { pub generated_frames: usize, } +/// Restore the session's generation mode on every speech exit, including +/// failures before the external-decode guard can be acquired. +struct SpeechEmbeddingsGuard(*mut skippy_ffi::Opaque); + +impl Drop for SpeechEmbeddingsGuard { + fn drop(&mut self) { + // SAFETY: the borrowed StageSession outlives this guard and owns the + // context; speech generation holds exclusive access to the session. + unsafe { skippy_ffi::llama_set_embeddings(self.0, false) }; + } +} + pub(crate) struct MediaProjector { pub(crate) raw: *mut skippy_ffi::MtmdContext, marker: String, @@ -142,6 +158,8 @@ impl StageModel { }) } + /// Generate bounded audio while restoring the session's normal decode mode + /// on success, cancellation, and native failure. Sessions remain exclusive. pub fn synthesize_speech( &self, session: &mut StageSession, @@ -152,7 +170,8 @@ impl StageModel { .media .as_ref() .ok_or_else(|| anyhow!("speech synthesis requires a configured projector"))?; - if !self.supports_speech_synthesis() { + let info = unsafe { skippy_ffi::mtmd_gen_audio_get_info(projector.raw) }; + if info.audio_type == skippy_ffi::MtmdGenAudioType::None { return Err(anyhow!( "configured projector does not support speech synthesis" )); @@ -193,16 +212,9 @@ impl StageModel { free_error(error); } } - struct EmbeddingsGuard(*mut skippy_ffi::Opaque); - impl Drop for EmbeddingsGuard { - fn drop(&mut self) { - unsafe { skippy_ffi::llama_set_embeddings(self.0, false) }; - } - } - session.reset()?; unsafe { skippy_ffi::llama_set_embeddings(lctx, true) }; - let _embeddings = EmbeddingsGuard(lctx); + let _embeddings_mode = SpeechEmbeddingsGuard(lctx); let mut guard_error = ptr::null_mut(); let status = unsafe { skippy_ffi::skippy_session_begin_external_decode(session.raw, &mut guard_error) @@ -907,3 +919,7 @@ mod tests { assert!(pcm_f32_to_s16le(&[0, 1, 2]).is_err()); } } + +#[cfg(test)] +#[path = "media/speech_session_tests.rs"] +mod speech_session_tests; diff --git a/crates/skippy-runtime/src/media/speech_session_tests.rs b/crates/skippy-runtime/src/media/speech_session_tests.rs new file mode 100644 index 0000000000..e38c985cd1 --- /dev/null +++ b/crates/skippy-runtime/src/media/speech_session_tests.rs @@ -0,0 +1,88 @@ +//! Real-projector lifecycle regressions, opt-in through the workload fixture. + +use super::*; +use crate::{ModelInfo, RuntimeConfig, TensorRole}; + +fn speech_fixture() -> Result> { + if std::env::var("SKIPPY_WORKLOAD_CLASS").as_deref() != Ok("speech_synthesis") { + return Ok(None); + } + let path = std::env::var("SKIPPY_WORKLOAD_MODEL").context("speech fixture model")?; + let projector = + std::env::var("SKIPPY_WORKLOAD_PROJECTOR").context("speech fixture projector")?; + let layer_end = ModelInfo::open(&path)? + .tensors()? + .into_iter() + .filter(|tensor| tensor.role == TensorRole::Layer) + .filter_map(|tensor| tensor.layer_index) + .max() + .context("speech fixture layers")? + + 1; + let config = RuntimeConfig { + layer_end, + ctx_size: 2048, + n_batch: Some(2048), + n_ubatch: Some(2048), + projector_path: Some(projector), + projector_use_gpu: Some(false), + kv_offload: Some(false), + op_offload: Some(false), + ..RuntimeConfig::default() + }; + StageModel::open(path, &config).map(Some) +} + +fn assert_generation_reusable(model: &StageModel, session: &mut StageSession) -> Result<()> { + session.reset()?; + let tokens = model.tokenize("The mesh is ready.", true)?; + let (last, prefix) = tokens.split_last().context("generation fixture tokens")?; + session.prefill_chunked(prefix)?; + // Plain prefill deliberately produces no logits. Decode the final token + // with an output row so this checks real generation, not a sampler fallback. + let token = session.decode_step(*last)?; + assert!(token >= 0, "normal generation must produce a valid token"); + assert_eq!(session.sample_current(None)?, token); + Ok(()) +} + +#[test] +fn speech_success_cancellation_and_native_failure_leave_session_reusable() -> Result<()> { + let Some(model) = speech_fixture()? else { + return Ok(()); + }; + let mut session = model.create_session()?; + let mut config = SpeechSynthesisConfig { + prompt: "Hello.".into(), + language: None, + top_k: 1, + top_p: 1.0, + seed: 42, + output_format: SpeechOutputFormat::Wav, + max_frames: 2, + }; + let cancelled = model + .synthesize_speech(&mut session, &config, || true) + .unwrap_err(); + assert!(cancelled.to_string().contains("cancelled"), "{cancelled:#}"); + assert_generation_reusable(&model, &mut session)?; + config.language = Some("not-a-supported-language".into()); + let rejected = model + .synthesize_speech(&mut session, &config, || false) + .unwrap_err(); + assert!( + rejected.to_string().contains("rejected the input"), + "{rejected:#}" + ); + assert_generation_reusable(&model, &mut session)?; + config.language = None; + let capped = model + .synthesize_speech(&mut session, &config, || false) + .unwrap_err(); + assert!(capped.to_string().contains("frame limit"), "{capped:#}"); + assert_generation_reusable(&model, &mut session)?; + config.max_frames = 512; + let audio = model.synthesize_speech(&mut session, &config, || false)?; + assert!(audio.sample_count > 0 && audio.bytes.len() > 44); + assert_generation_reusable(&model, &mut session)?; + Ok(()) +} diff --git a/crates/skippy-runtime/src/native.rs b/crates/skippy-runtime/src/native.rs index 7ca662f4ae..17c4ec375d 100644 --- a/crates/skippy-runtime/src/native.rs +++ b/crates/skippy-runtime/src/native.rs @@ -172,6 +172,7 @@ impl StageModel { present.then(|| raw.into()) } + /// Read the loaded model's ABI-validated workload, pooling, and output dimensions. pub fn workload_info(&self) -> Result { let mut raw = skippy_ffi::WorkloadInfoV1::default(); let mut error = ptr::null_mut(); diff --git a/crates/skippy-runtime/src/session.rs b/crates/skippy-runtime/src/session.rs index 4236bf36f5..1d3a7c59b8 100644 --- a/crates/skippy-runtime/src/session.rs +++ b/crates/skippy-runtime/src/session.rs @@ -87,6 +87,7 @@ impl StageSession { Ok(()) } + /// Produce one native pooled vector with a positive, caller-verified dimension. pub fn embed(&mut self, token_ids: &[i32], dimensions: usize) -> Result> { if dimensions == 0 { return Err(anyhow!("embedding dimensions must be greater than zero")); @@ -115,6 +116,7 @@ impl StageSession { Ok(output) } + /// Score a query/document pair and return the native template's consumed tokens. pub fn rerank(&mut self, query: &str, document: &str) -> Result<(f32, usize)> { let query = CString::new(query).context("rerank query contains an interior NUL byte")?; let document = @@ -137,6 +139,7 @@ impl StageSession { Ok((score, token_count)) } + /// Encode source tokens and reset decoder position, returning its first input token. pub fn encode_prompt(&mut self, token_ids: &[i32]) -> Result { let mut decoder_start_token = 0_i32; let mut error = ptr::null_mut(); diff --git a/crates/skippy-server/src/frontend/backend.rs b/crates/skippy-server/src/frontend/backend.rs index 4896b071b0..0078902fd5 100644 --- a/crates/skippy-server/src/frontend/backend.rs +++ b/crates/skippy-server/src/frontend/backend.rs @@ -1134,8 +1134,7 @@ impl OpenAiBackend for StageOpenAiBackend { ) -> OpenAiResult { self.ensure_model(&request.model)?; let info = self.ensure_local_workload(ModelWorkload::Embedding)?; - let expected_dimensions = usize::try_from(info.output_dimensions) - .map_err(|_| OpenAiError::backend("embedding dimensions exceed usize"))?; + let expected_dimensions = non_chat::embedding_output_dimensions(info.output_dimensions)?; if request .dimensions .is_some_and(|requested| requested != expected_dimensions) @@ -1190,19 +1189,7 @@ impl OpenAiBackend for StageOpenAiBackend { ) -> OpenAiResult { self.ensure_model(&request.model)?; self.ensure_local_workload(ModelWorkload::Rerank)?; - let prompt_tokens_estimate = request - .documents - .iter() - .filter_map(|document| document.text().ok()) - .map(|document| { - request - .query - .len() - .saturating_add(document.len()) - .div_ceil(3) - }) - .max() - .unwrap_or(1); + let prompt_tokens_estimate = non_chat::rerank_prompt_tokens_estimate(&request)?; let ids = generation_ids(OpenAiCacheHints::default(), None, &context); let cancellation = context.cancellation_token(); let query = request.query.clone(); diff --git a/crates/skippy-server/src/frontend/backend/non_chat.rs b/crates/skippy-server/src/frontend/backend/non_chat.rs index cc712a551d..300982f11f 100644 --- a/crates/skippy-server/src/frontend/backend/non_chat.rs +++ b/crates/skippy-server/src/frontend/backend/non_chat.rs @@ -1,5 +1,31 @@ use super::*; +/// Reject an invalid model descriptor before allocating any embedding sessions. +pub(super) fn embedding_output_dimensions(dimensions: u32) -> OpenAiResult { + if dimensions == 0 { + return Err(OpenAiError::backend( + "model did not report an embedding output dimension", + )); + } + usize::try_from(dimensions) + .map_err(|_| OpenAiError::backend("embedding dimensions exceed usize")) +} + +/// Validate the same documents that execution consumes before estimating admission. +pub(super) fn rerank_prompt_tokens_estimate(request: &RerankRequest) -> OpenAiResult { + request.validate()?; + request + .documents + .iter() + .map(|document| { + document + .text() + .map(|text| request.query.len().saturating_add(text.len()).div_ceil(3)) + }) + .try_fold(1, |estimate, next| next.map(|tokens| estimate.max(tokens))) +} + +/// Stop a native batch between items when its owning HTTP request is cancelled. pub(super) fn collect_workload_batch( items: I, cancellation: &openai_frontend::CancellationToken, @@ -21,6 +47,7 @@ where Ok(results) } +/// Preserve structured frontend errors while adding context to native failures. fn workload_error(error: anyhow::Error) -> OpenAiError { if let Some(openai_error) = error.downcast_ref::() { return openai_error.clone(); @@ -28,6 +55,7 @@ fn workload_error(error: anyhow::Error) -> OpenAiError { OpenAiError::backend(format!("workload execution failed: {error:#}")) } +/// Accept only the implemented default speaker; do not reinterpret voice as language. pub(super) fn validate_speech_voice(voice: &str) -> OpenAiResult<()> { if voice == "default" { Ok(()) @@ -40,6 +68,7 @@ pub(super) fn validate_speech_voice(voice: &str) -> OpenAiResult<()> { } impl StageOpenAiBackend { + /// Recognize complete local models in both standalone and embedded serving modes. pub(in crate::frontend) fn has_unsplit_full_model_topology(&self) -> bool { fn is_unsplit(config: &skippy_protocol::StageConfig) -> bool { config.stage_index == 0 @@ -60,6 +89,7 @@ impl StageOpenAiBackend { } } + /// Run audio transcription or English translation through a full-model projector. pub(super) async fn audio_to_text( &self, request: AudioTranscriptionRequest, @@ -128,6 +158,7 @@ impl StageOpenAiBackend { }) } + /// Reject split execution and mismatched native workload descriptors before work. pub(in crate::frontend) fn ensure_local_workload( &self, expected: ModelWorkload, @@ -160,6 +191,7 @@ impl StageOpenAiBackend { Ok(info) } + /// Tokenize text batches with the loaded vocabulary, preserving supplied token IDs. pub(super) fn prepare_embedding_inputs( &self, request: EmbeddingsRequest, @@ -188,6 +220,7 @@ impl StageOpenAiBackend { } } + /// Execute admitted blocking work with cancellation, slot ownership, and cleanup. pub(super) async fn run_local_workload( &self, context: OpenAiRequestContext, @@ -308,6 +341,31 @@ fn audio_text_user_message(instruction: String) -> openai_frontend::ChatMessage #[cfg(test)] mod tests { + #[test] + fn invalid_embedding_dimensions_are_rejected_at_admission() { + assert!( + super::embedding_output_dimensions(0) + .unwrap_err() + .body() + .error + .message + .contains("did not report") + ); + assert_eq!(super::embedding_output_dimensions(768).unwrap(), 768); + } + + #[test] + fn rerank_estimate_rejects_invalid_documents_before_workload_admission() { + let mut request: openai_frontend::RerankRequest = + serde_json::from_value(serde_json::json!({ + "model": "rank", "query": "query", "documents": ["text", {"title": "no text"}] + })) + .unwrap(); + assert!(super::rerank_prompt_tokens_estimate(&request).is_err()); + request.documents.pop(); + assert_eq!(super::rerank_prompt_tokens_estimate(&request).unwrap(), 3); + } + use super::{ audio_text_instruction, audio_text_user_message, audio_transcript_text, collect_workload_batch, validate_speech_voice, workload_error, diff --git a/scripts/check-skippy-workload-candidate.py b/scripts/check-skippy-workload-candidate.py index 7748612bf3..370f9e9049 100644 --- a/scripts/check-skippy-workload-candidate.py +++ b/scripts/check-skippy-workload-candidate.py @@ -14,6 +14,7 @@ def file_hash(path: Path) -> str: + """Hash artifact bytes incrementally without loading model-sized files in memory.""" digest = hashlib.sha256() with path.open("rb") as stream: for block in iter(lambda: stream.read(1024 * 1024), b""): @@ -22,7 +23,9 @@ def file_hash(path: Path) -> str: def source_identity(root: Path = ROOT) -> dict[str, str]: + """Bind the checkout head and all nonignored source changes to one identity.""" def git(*args: str) -> bytes: + """Read repository state without changing the checkout or index.""" return subprocess.check_output(["git", "-C", str(root), *args]) digest = hashlib.sha256(git("diff", "--binary", "HEAD", "--")) @@ -36,6 +39,7 @@ def git(*args: str) -> bytes: def producer_files(binary: Path, build_dir: Path, test_binary: Path) -> dict[str, Path]: + """Enumerate the candidate, native stamp, tools, and independent oracle closure.""" return { "candidate": binary, "test_binary": test_binary, @@ -47,6 +51,7 @@ def producer_files(binary: Path, build_dir: Path, test_binary: Path) -> dict[str def write_producer(output: Path, binary: Path, build_dir: Path, test_binary: Path, source_snapshot: Path) -> None: + """Write evidence only if source stayed unchanged throughout the producer build.""" source = source_identity() if source != json.loads(source_snapshot.read_text(encoding="utf-8")): raise RuntimeError("repository source changed while building workload producers") @@ -62,6 +67,7 @@ def write_producer(output: Path, binary: Path, build_dir: Path, test_binary: Pat def verify_producer(manifest: Path, binary: Path, build_dir: Path) -> None: + """Reject stale source, replaced executables, or a mismatched native test closure.""" payload = json.loads(manifest.read_text(encoding="utf-8")) if payload.get("schema_version") != 1 or payload.get("source") != source_identity(): raise RuntimeError("workload producer does not match the current repository head and worktree") @@ -75,6 +81,7 @@ def verify_producer(manifest: Path, binary: Path, build_dir: Path) -> None: def check_candidate(binary: Path, build_dir: Path) -> None: + """Require an executable newer than its stamped, statically linked native ABI.""" stamp = build_dir / ".mesh-llm-build-stamp" if not binary.is_file() or not os.access(binary, os.X_OK): raise RuntimeError(f"candidate executable is missing: {binary}") @@ -88,6 +95,7 @@ def check_candidate(binary: Path, build_dir: Path) -> None: def main() -> None: + """Dispatch source snapshots, producer creation, or read-only identity checks.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--candidate-binary", type=Path) parser.add_argument("--native-build-dir", type=Path) diff --git a/scripts/ci-openai-embeddings-smoke.py b/scripts/ci-openai-embeddings-smoke.py index 2ad1f8c6fb..4dea73f2de 100755 --- a/scripts/ci-openai-embeddings-smoke.py +++ b/scripts/ci-openai-embeddings-smoke.py @@ -8,8 +8,11 @@ import math import struct +from workload_fixtures import EMBEDDING_INPUTS + def main() -> None: + """Check batched numeric and base64 vectors through the official Python SDK.""" parser = argparse.ArgumentParser() parser.add_argument("--base-url", required=True) parser.add_argument("--model", required=True) @@ -23,10 +26,7 @@ def main() -> None: ) from exc client = OpenAI(api_key="mesh-llm-ci", base_url=args.base_url) - inputs = [ - "search_query: distributed inference", - "search_document: GPUs collaborate over a mesh", - ] + inputs = list(EMBEDDING_INPUTS) response = client.embeddings.create( model=args.model, input=inputs, diff --git a/scripts/ci-workload-monolithic-oracle.py b/scripts/ci-workload-monolithic-oracle.py index 59b8eb697e..2ab1785b87 100644 --- a/scripts/ci-workload-monolithic-oracle.py +++ b/scripts/ci-workload-monolithic-oracle.py @@ -45,6 +45,7 @@ def request_json(base_url: str, path: str, payload: dict[str, object]) -> dict: def vectors(response: dict, expected_count: int) -> list[list[float]]: + """Validate exact embedding cardinality, indexes, and finite nonempty vectors.""" rows = response.get("data") if not isinstance(rows, list) or len(rows) != expected_count: raise RuntimeError("embedding oracle response has the wrong batch size") @@ -62,6 +63,7 @@ def vectors(response: dict, expected_count: int) -> list[list[float]]: def compare_embeddings(candidate: dict, reference: dict, expected_count: int = len(EMBEDDING_INPUTS)) -> str: + """Require equal dimensions and bounded numeric disagreement for every input.""" candidate_vectors = vectors(candidate, expected_count) reference_vectors = vectors(reference, expected_count) max_delta = 0.0 @@ -94,6 +96,7 @@ def compare_embeddings(candidate: dict, reference: dict, expected_count: int = l def run_embedding_oracle(candidate_url: str, oracle_url: str, model: str) -> str: + """Compare the shared text fixture in both batched and single-input execution.""" payload = {"model": model, "input": list(EMBEDDING_INPUTS), "encoding_format": "float"} candidate = request_json(candidate_url, "/embeddings", payload) reference = request_json(oracle_url, "/embeddings", payload) @@ -118,6 +121,7 @@ def run_embedding_oracle(candidate_url: str, oracle_url: str, model: str) -> str def indexed_scores(response: dict) -> dict[int, float]: + """Reject missing, duplicate, or invalid rerank document indexes and scores.""" rows = response.get("results") if not isinstance(rows, list) or len(rows) != len(RERANK_DOCUMENTS): raise RuntimeError("rerank oracle response has the wrong document count") @@ -136,6 +140,7 @@ def indexed_scores(response: dict) -> dict[int, float]: def compare_rerank(candidate: dict, reference: dict) -> str: + """Require matching relevance order and bounded score differences per document.""" candidate_scores = indexed_scores(candidate) reference_scores = indexed_scores(reference) max_delta = max( @@ -154,6 +159,7 @@ def compare_rerank(candidate: dict, reference: dict) -> str: def completion_text(response: dict) -> str: + """Extract exactly one nonempty completion and normalize only its whitespace.""" choices = response.get("choices") if not isinstance(choices, list) or len(choices) != 1: raise RuntimeError("encoder-decoder oracle response has invalid choices") @@ -164,6 +170,7 @@ def completion_text(response: dict) -> str: def compare_encoder_decoder(candidate: dict, reference: dict) -> str: + """Require exact normalized completion parity with the independent reference.""" candidate_text = completion_text(candidate) reference_text = completion_text(reference) if candidate_text != reference_text: diff --git a/scripts/skippy-family-battery.sh b/scripts/skippy-family-battery.sh index 5eb70d4d80..b9e0deef89 100755 --- a/scripts/skippy-family-battery.sh +++ b/scripts/skippy-family-battery.sh @@ -4,10 +4,9 @@ set -euo pipefail # Supported-families certification battery (issue #1434; tiers dropped 2026-08-25). # # Causal-generation rows get core split certification: single-step, chain, -# and state-handoff lanes. Non-chat rows get a certified class-specific smoke -# lane through the Skippy runtime and OpenAI-compatible frontend plus an -# independent local-monolithic oracle lane. These classes deliberately fail -# closed if asked to stage. Models with +# and state-handoff lanes. Certified non-chat rows get class-specific smoke +# and independent local-monolithic oracle lanes through the Skippy runtime; +# these classes deliberately fail closed if asked to stage. Models with # MTP/NextN tensors require the native draft sideband and verify it against the # target in the correctness lanes. Dense causal rows run them at the first, # midpoint, and last interior cuts. Hybrid/recurrent rows (sweep_period > 0) @@ -926,16 +925,6 @@ run_workload_certify() { fi if (( certified == 1 )); then command+=(--require-oracle) - if (( DRY_RUN == 1 )); then - echo "==> workload certification: family=$family class=$model_class lanes=$lane_csv model=$(basename "$target")" - printf '%q ' "${command[@]}" - printf '\n' - return 0 - fi - if (( oracle_requested != 1 )); then - echo "certified workload $family ($model_class) requires a class-appropriate local-monolithic oracle executable" >&2 - exit 1 - fi fi echo "==> workload certification: family=$family class=$model_class lanes=$lane_csv model=$(basename "$target")" if (( DRY_RUN == 1 )); then @@ -944,10 +933,15 @@ run_workload_certify() { return 0 fi exit_code=0 - "$ROOT/scripts/run-command-with-timeout.py" \ - --seconds "$cert_timeout" \ - --label "workload certification $family ($model_class)" \ - -- "${command[@]}" >"$log_path" 2>&1 || exit_code=$? + if (( certified == 1 && oracle_requested != 1 )); then + echo "certified workload $family ($model_class) requires a class-appropriate local-monolithic oracle executable" | tee "$log_path" >&2 + exit_code=1 + else + "$ROOT/scripts/run-command-with-timeout.py" \ + --seconds "$cert_timeout" \ + --label "workload certification $family ($model_class)" \ + -- "${command[@]}" >"$log_path" 2>&1 || exit_code=$? + fi if (( certified == 1 && exit_code == 0 )); then local verify_command=(python3 "$ROOT/scripts/verify-workload-oracle-evidence.py" \ --evidence "$cert_run_dir/workload-oracle-evidence.json" \ diff --git a/scripts/skippy-ocr-asr-oracle.py b/scripts/skippy-ocr-asr-oracle.py index 5f22fa9fc6..aa98d6770e 100644 --- a/scripts/skippy-ocr-asr-oracle.py +++ b/scripts/skippy-ocr-asr-oracle.py @@ -32,6 +32,7 @@ def normalized_text(value: object, source: str) -> str: + """Normalize Unicode, case, and punctuation while rejecting empty output.""" if not isinstance(value, str): raise RuntimeError(f"{source} returned no text") normalized = unicodedata.normalize("NFKC", value).casefold() @@ -42,6 +43,7 @@ def normalized_text(value: object, source: str) -> str: def transcription_text(value: object, source: str) -> str: + """Remove only known presentation labels and reject refusals masquerading as ASR.""" text = normalized_text(value, source) # The two frontends add different presentational labels around the same # transcript. Strip only these exact known prefixes, never content words. @@ -56,6 +58,7 @@ def transcription_text(value: object, source: str) -> str: def compare_text(candidate: object, reference: object, expected: str | None, *, transcript: bool = False) -> str: + """Require exact normalized parity and, when provided, the independent fixture label.""" normalizer = transcription_text if transcript else normalized_text candidate_text = normalizer(candidate, "candidate") reference_text = normalizer(reference, "monolithic reference") diff --git a/scripts/skippy-tts-oracle.py b/scripts/skippy-tts-oracle.py index 3cd8c467c8..9825b4d574 100644 --- a/scripts/skippy-tts-oracle.py +++ b/scripts/skippy-tts-oracle.py @@ -26,7 +26,9 @@ SEED = 7 TOP_K = 20 TOP_P = 0.8 -MAX_FRAMES = 32 +# Compare complete utterances; a short frame cap would only compare two +# identically truncated clips and is now rejected by the candidate runtime. +MAX_FRAMES = 512 CONTEXT_SIZE = 2048 MAX_RELATIVE_RMS_ERROR = 0.02 MIN_WAVEFORM_COSINE = 0.9995 @@ -34,6 +36,7 @@ def require_pinned_cpu_oracle(oracle_cli: Path) -> str: + """Verify the CPU reference build options and return its prepared patch identity.""" if ( oracle_cli.name != "llama-tts" or not oracle_cli.is_file() @@ -61,6 +64,7 @@ def require_pinned_cpu_oracle(oracle_cli: Path) -> str: def require_candidate_cpu_static_build(build_dir: Path, patched_sha: str) -> None: + """Require the candidate's static CPU ABI to match the independent reference.""" stamp_path = build_dir / ".mesh-llm-build-stamp" if not stamp_path.is_file(): raise RuntimeError("TTS candidate lacks a pinned static CPU build stamp") @@ -78,6 +82,7 @@ def require_candidate_cpu_static_build(build_dir: Path, patched_sha: str) -> Non def run_logged(command: list[str], log_path: Path, *, env: dict[str, str] | None = None) -> None: + """Bound one oracle process and retain its combined output on success or failure.""" with log_path.open("w", encoding="utf-8") as log: try: result = subprocess.run( @@ -97,6 +102,7 @@ def run_logged(command: list[str], log_path: Path, *, env: dict[str, str] | None def read_pcm16_wav(path: Path) -> tuple[int, int, array]: + """Decode nonempty PCM16 WAV data with explicit format and endian validation.""" try: with wave.open(str(path), "rb") as audio: sample_rate = audio.getframerate() @@ -116,6 +122,7 @@ def read_pcm16_wav(path: Path) -> tuple[int, int, array]: def compare_wavs(candidate_path: Path, oracle_path: Path) -> dict[str, float | int]: + """Check exact audio dimensions and bounded, nonsilent waveform disagreement.""" candidate_rate, candidate_channels, candidate = read_pcm16_wav(candidate_path) oracle_rate, oracle_channels, oracle = read_pcm16_wav(oracle_path) if candidate_rate != oracle_rate or candidate_channels != oracle_channels: @@ -153,6 +160,7 @@ def compare_wavs(candidate_path: Path, oracle_path: Path) -> dict[str, float | i def sha256(path: Path) -> str: + """Stream an artifact digest for identity-bound certification evidence.""" digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): @@ -161,6 +169,7 @@ def sha256(path: Path) -> str: def candidate_test_command(env: dict[str, str]) -> list[str]: + """Select the verified prebuilt test binary or the explicit local test fallback.""" manifest = env.get("SKIPPY_WORKLOAD_PRODUCER_MANIFEST") if not manifest: return ["cargo", "test", "--manifest-path", str(ROOT / "Cargo.toml"), @@ -184,6 +193,7 @@ def candidate_test_command(env: dict[str, str]) -> list[str]: def run_oracle(args: argparse.Namespace) -> dict[str, object]: + """Generate both deterministic utterances and bind measured PCM parity to inputs.""" oracle_cli = Path(args.oracle_cli).resolve() model_path = Path(args.model_path).resolve() projector_path = Path(args.projector_path).resolve() @@ -266,6 +276,7 @@ def run_oracle(args: argparse.Namespace) -> dict[str, object]: def main() -> None: + """Run the selected TTS comparison and persist its measured evidence.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--oracle-cli", required=True) parser.add_argument("--model-path", required=True) diff --git a/scripts/skippy-workload-certify.sh b/scripts/skippy-workload-certify.sh index 4194df38f0..f7013770b0 100755 --- a/scripts/skippy-workload-certify.sh +++ b/scripts/skippy-workload-certify.sh @@ -20,8 +20,8 @@ usage() { usage: scripts/skippy-workload-certify.sh --class CLASS --lane LANE --model-path PATH --model-id ID --work-dir PATH [--projector-path PATH] [--oracle-server PATH] [--oracle-completion PATH] [--oracle-tts PATH] - [--startup-timeout-secs SECONDS] [--require-oracle] # fail closed unless the class-appropriate oracle is selected + [--startup-timeout-secs SECONDS] # per-server readiness deadline (default: 180) [--skip-build] # oracle runs require a prebuilt SKIPPY_WORKLOAD_PRODUCER_MANIFEST EOF } @@ -37,8 +37,8 @@ while (( $# > 0 )); do --oracle-server) ORACLE_SERVER="$2"; shift ;; --oracle-completion) ORACLE_COMPLETION="$2"; shift ;; --oracle-tts) ORACLE_TTS="$2"; shift ;; - --startup-timeout-secs) STARTUP_TIMEOUT_SECS="$2"; shift ;; --require-oracle) ORACLE_REQUIRED=1 ;; + --startup-timeout-secs) STARTUP_TIMEOUT_SECS="$2"; shift ;; --skip-build) SKIP_BUILD=1 ;; -h|--help) usage; exit 0 ;; *) echo "unknown option: $1" >&2; usage; exit 1 ;; @@ -46,8 +46,8 @@ while (( $# > 0 )); do shift done -if [[ ! "$STARTUP_TIMEOUT_SECS" =~ ^[1-9][0-9]*$ ]]; then - echo "--startup-timeout-secs must be a positive integer" >&2 +if [[ ! "$STARTUP_TIMEOUT_SECS" =~ ^[1-9][0-9]{0,4}$ ]] || (( STARTUP_TIMEOUT_SECS > 86400 )); then + echo "--startup-timeout-secs must be a positive integer between 1 and 86400 seconds" >&2 exit 1 fi @@ -142,6 +142,12 @@ if [[ -n "$ORACLE_TTS" ]]; then require_pinned_cpu_oracle "$ORACLE_TTS" llama-tts 'cmake-arg=-DLLAMA_BUILD_TOOLS=ON' fi +SDK_PYTHON="${SKIPPY_WORKLOAD_SDK_PYTHON:-python3}" +if [[ "$MODEL_CLASS" == "embedding" ]] && ! "$SDK_PYTHON" -c 'import openai' >/dev/null 2>&1; then + echo "official openai-python SDK smoke requires the openai package in $SDK_PYTHON" >&2 + exit 1 +fi + mkdir -p "$WORK_DIR" EVIDENCE_PATH="$WORK_DIR/workload-oracle-evidence.json" COMPARISON_LOG="$WORK_DIR/workload-oracle-comparison.txt" @@ -270,20 +276,28 @@ cleanup() { } trap cleanup EXIT -for (( attempt = 0; attempt < STARTUP_TIMEOUT_SECS; attempt++ )); do - if ! kill -0 "$SERVER_PID" >/dev/null 2>&1; then - echo "$MODEL_CLASS OpenAI server exited early" >&2 - sed -n '1,240p' "$SERVER_LOG" >&2 - exit 1 - fi - if curl -fsS --max-time 1 "http://127.0.0.1:$PORT/v1/models" 2>/dev/null \ - | jq -e --arg model "$MODEL_ID" '.data[]? | select(.id == $model)' >/dev/null 2>&1; then - break - fi - sleep 1 -done -curl -fsS --max-time 2 "http://127.0.0.1:$PORT/v1/models" \ - | jq -e --arg model "$MODEL_ID" '.data[]? | select(.id == $model)' >/dev/null +# Both processes receive the same planned wall-clock startup budget, including +# time spent probing the endpoint. An early exit retains the owning log. +wait_for_workload_server() { + local pid="$1" port="$2" log="$3" label="$4" + local deadline=$((SECONDS + STARTUP_TIMEOUT_SECS)) + while (( SECONDS < deadline )); do + if ! kill -0 "$pid" >/dev/null 2>&1; then + echo "$MODEL_CLASS $label exited early" >&2 + tail -80 "$log" >&2 + return 1 + fi + if curl -fsS --max-time 1 "http://127.0.0.1:$port/v1/models" 2>/dev/null \ + | jq -e --arg model "$MODEL_ID" '.data[]? | select(.id == $model)' >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + echo "$MODEL_CLASS $label was not ready within $STARTUP_TIMEOUT_SECS seconds" >&2 + tail -80 "$log" >&2 + return 1 +} +wait_for_workload_server "$SERVER_PID" "$PORT" "$SERVER_LOG" "OpenAI server" python3 "$ROOT/scripts/ci-openai-workload-smoke.py" \ --base-url "http://127.0.0.1:$PORT/v1" \ --model "$MODEL_ID" \ @@ -309,20 +323,7 @@ if [[ -n "$ORACLE_SERVER" ]]; then ORACLE_LOG="$WORK_DIR/workload-monolithic-oracle-server.log" "$ORACLE_SERVER" "${ORACLE_ARGS[@]}" >"$ORACLE_LOG" 2>&1 & ORACLE_PID="$!" - for (( attempt = 0; attempt < STARTUP_TIMEOUT_SECS; attempt++ )); do - if ! kill -0 "$ORACLE_PID" >/dev/null 2>&1; then - echo "$MODEL_CLASS monolithic oracle server exited early" >&2 - tail -80 "$ORACLE_LOG" >&2 - exit 1 - fi - if curl -fsS --max-time 1 "http://127.0.0.1:$ORACLE_PORT/v1/models" 2>/dev/null \ - | jq -e --arg model "$MODEL_ID" '.data[]? | select(.id == $model)' >/dev/null 2>&1; then - break - fi - sleep 1 - done - curl -fsS --max-time 2 "http://127.0.0.1:$ORACLE_PORT/v1/models" \ - | jq -e --arg model "$MODEL_ID" '.data[]? | select(.id == $model)' >/dev/null + wait_for_workload_server "$ORACLE_PID" "$ORACLE_PORT" "$ORACLE_LOG" "monolithic oracle server" if [[ "$MODEL_CLASS" =~ ^(ocr|speech_recognition)$ ]]; then ORACLE_MEDIA_PATH="$MEDIA_PATH" if [[ "$MODEL_CLASS" == "ocr" ]]; then @@ -363,6 +364,12 @@ if [[ -n "$ORACLE_TTS" ]]; then --work-dir "$WORK_DIR" | tee "$COMPARISON_LOG" fi +if [[ "$MODEL_CLASS" == "embedding" ]]; then + "$SDK_PYTHON" "$ROOT/scripts/ci-openai-embeddings-smoke.py" \ + --base-url "http://127.0.0.1:$PORT/v1" \ + --model "$MODEL_ID" +fi + if [[ -n "$ORACLE_SERVER" || -n "$ORACLE_COMPLETION" || -n "$ORACLE_TTS" ]]; then ORACLE_EXECUTABLE="${ORACLE_SERVER:-${ORACLE_COMPLETION:-$ORACLE_TTS}}" evidence_command=(python3 "$ROOT/scripts/write-workload-oracle-evidence.py" @@ -381,14 +388,3 @@ if [[ -n "$ORACLE_SERVER" || -n "$ORACLE_COMPLETION" || -n "$ORACLE_TTS" ]]; the fi "${evidence_command[@]}" fi - -if [[ "$MODEL_CLASS" == "embedding" ]]; then - SDK_PYTHON="${SKIPPY_WORKLOAD_SDK_PYTHON:-python3}" - "$SDK_PYTHON" -c 'import openai' >/dev/null 2>&1 || { - echo "official openai-python SDK smoke requires the openai package in $SDK_PYTHON" >&2 - exit 1 - } - "$SDK_PYTHON" "$ROOT/scripts/ci-openai-embeddings-smoke.py" \ - --base-url "http://127.0.0.1:$PORT/v1" \ - --model "$MODEL_ID" -fi diff --git a/scripts/tests/test_skippy_ocr_asr_oracle.py b/scripts/tests/test_skippy_ocr_asr_oracle.py index 005da72659..4357656406 100644 --- a/scripts/tests/test_skippy_ocr_asr_oracle.py +++ b/scripts/tests/test_skippy_ocr_asr_oracle.py @@ -51,6 +51,11 @@ def test_ocr_requires_both_parity_and_known_text(self): with self.assertRaisesRegex(RuntimeError, "does not exactly match independently known"): oracle.compare_text("MESH 42 extra", "Mesh 42 extra.", "MESH 42") + def test_matching_incorrect_text_cannot_pass_by_containing_the_label(self): + for text in ("not mesh 42", "mesh 42 unrelated text", "mesh 42 mesh 42"): + with self.subTest(text=text), self.assertRaisesRegex(RuntimeError, "does not exactly match independently known"): + oracle.compare_text(text, text, "MESH 42") + def test_asr_unlabeled_fixture_does_not_claim_accuracy(self): detail = oracle.compare_text("The mesh is ready.", "the mesh is ready", None) self.assertIn("no accuracy claim", detail) diff --git a/scripts/tests/test_skippy_static_link.py b/scripts/tests/test_skippy_static_link.py index 1ba17e8ca5..8b2920df50 100644 --- a/scripts/tests/test_skippy_static_link.py +++ b/scripts/tests/test_skippy_static_link.py @@ -104,13 +104,27 @@ def test_backend_cache_mismatch_fails_closed_despite_stale_archive(self) -> None def test_unselected_backend_cache_mismatch_fails_closed(self) -> None: result = self._run("cpu", {"GGML_CUDA": "ON"}) self.assertNotEqual(0, result.returncode) - self.assertIn("unselected backend requires GGML_CUDA=OFF", result.stderr) + self.assertIn("staged backend mismatch: GGML_CUDA=ON", result.stderr) def test_crlf_cache_values_are_recognized(self) -> None: result = self._run_with_newline("metal", {"GGML_METAL": "ON"}, "\r\n") self.assertEqual(0, result.returncode, result.stderr) self.assertIn("cargo:rustc-link-lib=static=ggml-metal", result.stdout) + def test_enabled_staged_accelerator_requires_matching_selected_backend(self) -> None: + for key in ("GGML_CUDA", "GGML_HIP", "GGML_VULKAN", "GGML_METAL"): + with self.subTest(key=key): + result = self._run("cpu", {key: "ON"}) + self.assertNotEqual(0, result.returncode) + self.assertIn(f"staged backend mismatch: {key}=ON", result.stderr) + + def test_cache_boolean_accepts_crlf_and_surrounding_whitespace(self) -> None: + for value in ("ON\r", " TRUE \r", " 1 "): + with self.subTest(value=value): + result = self._run("metal", {"GGML_METAL": value}) + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn("cargo:rustc-link-lib=static=ggml-metal", result.stdout) + if __name__ == "__main__": unittest.main() diff --git a/scripts/tests/test_skippy_workload_certify.py b/scripts/tests/test_skippy_workload_certify.py index 5f98e2e510..6549523152 100644 --- a/scripts/tests/test_skippy_workload_certify.py +++ b/scripts/tests/test_skippy_workload_certify.py @@ -1,6 +1,7 @@ from __future__ import annotations from pathlib import Path +import os import subprocess import tempfile import unittest @@ -31,6 +32,28 @@ def test_help_documents_the_typed_certification_inputs(self) -> None: self.assertIn("--oracle-tts PATH", result.stderr) self.assertIn("--startup-timeout-secs SECONDS", result.stderr) self.assertIn("--require-oracle", result.stderr) + self.assertIn("--startup-timeout-secs", result.stderr) + + def test_startup_deadline_rejects_invalid_values_before_execution(self) -> None: + for value in ("0", "-1", "1.5", "01", "86401", "abc"): + with self.subTest(value=value): + result = self._run("--startup-timeout-secs", value) + self.assertEqual(1, result.returncode) + self.assertIn("must be a positive integer", result.stderr) + + def test_missing_embedding_sdk_fails_before_model_execution(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + model = Path(temp_dir) / "model.gguf" + model.touch() + result = subprocess.run( + [str(RUNNER), "--class", "embedding", "--lane", "embedding-smoke", + "--model-path", str(model), "--model-id", "fixture", "--work-dir", temp_dir, + "--skip-build"], cwd=ROOT, text=True, capture_output=True, check=False, + env={**os.environ, "SKIPPY_WORKLOAD_SDK_PYTHON": str(Path(temp_dir) / "missing-python")}, + ) + self.assertEqual(1, result.returncode) + self.assertIn("official openai-python SDK smoke requires", result.stderr) + self.assertFalse((Path(temp_dir) / "workload-oracle-evidence.json").exists()) def test_startup_timeout_must_be_positive(self) -> None: result = self._run("--startup-timeout-secs", "0") diff --git a/scripts/tests/test_verify_workload_oracle_evidence.py b/scripts/tests/test_verify_workload_oracle_evidence.py index 1e9601db6e..c527e03366 100644 --- a/scripts/tests/test_verify_workload_oracle_evidence.py +++ b/scripts/tests/test_verify_workload_oracle_evidence.py @@ -44,28 +44,29 @@ def setUp(self) -> None: "comparison": "embedding local-monolithic oracle passed: max_abs_delta=0, min_cosine=1", } - def run_verifier(self) -> subprocess.CompletedProcess[str]: + def run_verifier(self, model_class: str = "embedding", *extra: str) -> subprocess.CompletedProcess[str]: return subprocess.run( [ "python3", str(VERIFIER), "--evidence", str(self.evidence), - "--class", "embedding", "--smoke-lane", "embedding-smoke", + "--class", model_class, "--smoke-lane", "embedding-smoke", "--oracle-lane", "embedding-oracle", "--model-id", "fixture", "--model-path", str(self.model), "--candidate-executable", str(self.candidate), "--oracle-executable", str(self.oracle), "--pinned-patch-sha", "a" * 40, + *extra, ], cwd=ROOT, text=True, capture_output=True, check=False, ) - def run_writer(self, comparison: str) -> subprocess.CompletedProcess[str]: + def run_writer(self, comparison: str, lane: str = "embedding-smoke") -> subprocess.CompletedProcess[str]: comparison_log = Path(self.temp_dir.name) / "comparison.txt" comparison_log.write_text(comparison + "\n", encoding="utf-8") return subprocess.run( [ "python3", str(WRITER), "--output", str(self.evidence), "--comparison-log", str(comparison_log), "--class", "embedding", - "--smoke-lane", "embedding-smoke", "--model-id", "fixture", + "--smoke-lane", lane, "--model-id", "fixture", "--model-sha256", sha256(self.model), "--candidate-executable", str(self.candidate), "--oracle-executable", str(self.oracle), @@ -85,6 +86,39 @@ def test_smoke_only_log_never_writes_oracle_evidence(self) -> None: self.assertEqual(1, written.returncode) self.assertFalse(self.evidence.exists()) + def test_writer_rejects_missing_suffix_and_replaces_only_final_suffix(self) -> None: + for lane in ("embedding", "embedding-smoke-extra", "embedding-oracle"): + with self.subTest(lane=lane): + result = self.run_writer(self.body["comparison"], lane) + self.assertEqual(1, result.returncode) + self.assertIn("must end with '-smoke'", result.stderr) + self.assertFalse(self.evidence.exists()) + result = self.run_writer(self.body["comparison"], "fixture-smoke-embedding-smoke") + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual("fixture-smoke-embedding-oracle", json.loads(self.evidence.read_text())["oracle_lane"]) + + def test_projector_classes_require_independently_supplied_projector(self) -> None: + for model_class in ("ocr", "speech_synthesis", "speech_recognition"): + with self.subTest(model_class=model_class): + self.body["class"] = model_class + self.evidence.write_text(json.dumps(self.body), encoding="utf-8") + result = self.run_verifier(model_class) + self.assertEqual(1, result.returncode) + self.assertIn("requires a projector path", result.stderr) + + def test_projector_digest_is_verified_against_local_bytes(self) -> None: + projector = Path(self.temp_dir.name) / "projector.gguf" + projector.write_bytes(b"projector") + self.body.update({"class": "ocr", "projector_sha256": sha256(projector), + "comparison": "ocr local-monolithic oracle passed: exact text"}) + self.evidence.write_text(json.dumps(self.body), encoding="utf-8") + result = self.run_verifier("ocr", "--projector-path", str(projector)) + self.assertEqual(0, result.returncode, result.stderr) + projector.write_bytes(b"different projector") + result = self.run_verifier("ocr", "--projector-path", str(projector)) + self.assertEqual(1, result.returncode) + self.assertIn("projector_sha256 does not match", result.stderr) + def test_matching_explicit_evidence_is_accepted(self) -> None: self.evidence.write_text(json.dumps(self.body), encoding="utf-8") result = self.run_verifier() diff --git a/scripts/tests/test_workload_lane_execution.py b/scripts/tests/test_workload_lane_execution.py new file mode 100644 index 0000000000..3c685ffda2 --- /dev/null +++ b/scripts/tests/test_workload_lane_execution.py @@ -0,0 +1,102 @@ +"""Exercise the real shell lane functions without loading models or building.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import subprocess +import tempfile +import time +import unittest + +ROOT = Path(__file__).resolve().parents[2] + + +def shell_function(script: str, name: str) -> str: + """Load one top-level function verbatim, excluding the script's entrypoint.""" + source = (ROOT / "scripts" / script).read_text(encoding="utf-8") + start = source.index(f"{name}() {{\n") + end = source.index("\n}\n", start) + 3 + return source[start:end] + + +class WorkloadLaneExecutionTests(unittest.TestCase): + def run_lane(self, dry_run: bool) -> tuple[subprocess.CompletedProcess[str], list[dict]]: + with tempfile.TemporaryDirectory() as directory: + env = {key: value for key, value in os.environ.items() + if not key.startswith("SKIPPY_WORKLOAD_ORACLE_")} + env.update({"ROOT": str(ROOT), "CERT_DIR": directory, + "RESULTS_JSONL": str(Path(directory) / "results.jsonl"), + "DRY_RUN": str(int(dry_run))}) + script = "\n".join([ + "set -euo pipefail", "TOTAL=0; CERT_FAILURE_COUNT=0; FAILURES=()", + shell_function("skippy-family-battery.sh", "slugify"), + "cert_timeout_for_startup() { printf 1800; }", + shell_function("skippy-family-battery.sh", "run_workload_certify"), + 'run_workload_certify first embedding /unused/model.gguf fixture rev 600 1024 embedding-smoke,embedding-oracle ""', + 'run_workload_certify second rerank /unused/model.gguf fixture rev 900 1024 rerank-smoke,rerank-oracle ""', + 'printf "counts=%s,%s\\n" "$TOTAL" "$CERT_FAILURE_COUNT"', + ]) + result = subprocess.run(["bash", "-c", script], env=env, cwd=ROOT, + capture_output=True, text=True, check=False, timeout=15) + path = Path(env["RESULTS_JSONL"]) + # jq emits pretty-printed objects separated by whitespace. + rows = [] + remaining = path.read_text() if path.exists() else "" + decoder = json.JSONDecoder() + while remaining.strip(): + row, end = decoder.raw_decode(remaining.lstrip()) + rows.append(row) + remaining = remaining.lstrip()[end:] + return result, rows + + def test_dry_run_prints_both_certified_rows_without_oracles(self) -> None: + result, rows = self.run_lane(True) + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn("--startup-timeout-secs 600", result.stdout) + self.assertIn("--startup-timeout-secs 900", result.stdout) + self.assertEqual(2, result.stdout.count("--require-oracle")) + self.assertIn("counts=2,0", result.stdout) + self.assertEqual([], rows) + + def test_missing_oracle_records_both_failures_and_continues(self) -> None: + result, rows = self.run_lane(False) + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn("counts=2,2", result.stdout) + self.assertEqual(["first", "second"], [row["family"] for row in rows]) + for row in rows: + self.assertEqual(1, row["exit_code"]) + self.assertEqual(2, len(row["outcomes"])) + self.assertTrue(all(lane["status"] == "fail" for lane in row["outcomes"])) + + def test_readiness_uses_deadline_for_each_server_and_rejects_dead_process(self) -> None: + function = shell_function("skippy-workload-certify.sh", "wait_for_workload_server") + for label in ("OpenAI server", "monolithic oracle server"): + with self.subTest(label=label), tempfile.TemporaryDirectory() as directory: + log = Path(directory) / "server.log" + log.write_text("fixture startup log\n") + env = {**os.environ, "STARTUP_TIMEOUT_SECS": "1", "MODEL_CLASS": "embedding", + "MODEL_ID": "fixture", "SERVER_LOG": str(log), "LABEL": label} + prefix = "set -euo pipefail\n" + function + "\n" + start = time.monotonic() + result = subprocess.run(["bash", "-c", prefix + + 'curl() { return 1; }; wait_for_workload_server $$ 1 "$SERVER_LOG" "$LABEL"'], + env=env, capture_output=True, text=True, check=False, timeout=5) + self.assertEqual(1, result.returncode) + self.assertLess(time.monotonic() - start, 3) + self.assertIn(f"{label} was not ready within 1 seconds", result.stderr) + self.assertIn("fixture startup log", result.stderr) + result = subprocess.run(["bash", "-c", prefix + + 'wait_for_workload_server 99999999 1 "$SERVER_LOG" "$LABEL"'], + env=env, capture_output=True, text=True, check=False, timeout=5) + self.assertEqual(1, result.returncode) + self.assertIn(f"{label} exited early", result.stderr) + result = subprocess.run(["bash", "-c", prefix + + 'curl() { printf \'{"data":[{"id":"fixture"}]}\'; }; wait_for_workload_server $$ 1 "$SERVER_LOG" "$LABEL"'], + env=env, capture_output=True, text=True, check=False, timeout=5) + self.assertEqual(0, result.returncode, result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/verify-workload-oracle-evidence.py b/scripts/verify-workload-oracle-evidence.py index 8dba7bcb60..dedf10e72e 100644 --- a/scripts/verify-workload-oracle-evidence.py +++ b/scripts/verify-workload-oracle-evidence.py @@ -21,6 +21,7 @@ def sha256(path: Path) -> str: + """Hash independently supplied artifact bytes for comparison with evidence.""" digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): @@ -29,7 +30,8 @@ def sha256(path: Path) -> str: def verify(args: argparse.Namespace) -> None: - if args.model_class in {"ocr", "speech_synthesis", "speech_recognition"} and args.projector_path is None: + """Reject missing prerequisites or evidence not bound to these exact inputs.""" + if args.model_class in {"ocr", "speech_synthesis", "speech_recognition"} and not args.projector_path: raise ValueError(f"{args.model_class} oracle evidence requires a projector path") evidence = json.loads(args.evidence.read_text(encoding="utf-8")) if not isinstance(evidence, dict): @@ -62,6 +64,7 @@ def verify(args: argparse.Namespace) -> None: def main() -> int: + """Return success only when the recorded comparator pass matches this run.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--evidence", required=True, type=Path) parser.add_argument("--class", dest="model_class", required=True, choices=ORACLE_EXECUTABLE) diff --git a/scripts/write-workload-oracle-evidence.py b/scripts/write-workload-oracle-evidence.py index 4b7b85f7b4..ee24a7c665 100644 --- a/scripts/write-workload-oracle-evidence.py +++ b/scripts/write-workload-oracle-evidence.py @@ -11,6 +11,7 @@ def sha256(path: Path) -> str: + """Hash a model sidecar or executable without buffering the whole artifact.""" digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): @@ -19,8 +20,9 @@ def sha256(path: Path) -> str: def write_evidence(args: argparse.Namespace) -> None: + """Bind an observed comparator pass to its lane and executable identities.""" if not args.smoke_lane.endswith("-smoke"): - raise ValueError("smoke lane must end with '-smoke'") + raise ValueError(f"smoke lane must end with '-smoke': {args.smoke_lane}") lines = [ line.strip() for line in args.comparison_log.read_text(encoding="utf-8").splitlines() @@ -34,7 +36,7 @@ def write_evidence(args: argparse.Namespace) -> None: "status": "pass", "class": args.model_class, "smoke_lane": args.smoke_lane, - "oracle_lane": args.smoke_lane.removesuffix("-smoke") + "-oracle", + "oracle_lane": f"{args.smoke_lane.removesuffix('-smoke')}-oracle", "model_id": args.model_id, "model_sha256": args.model_sha256, "projector_sha256": sha256(args.projector_path) if args.projector_path else None, @@ -55,6 +57,7 @@ def write_evidence(args: argparse.Namespace) -> None: def main() -> int: + """Persist a verified comparator result, returning failure for incomplete evidence.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--output", required=True, type=Path) parser.add_argument("--comparison-log", required=True, type=Path) diff --git a/third_party/llama.cpp/patches/0028-skippy-validate-sampling-across-all-execution-bounda.patch b/third_party/llama.cpp/patches/0028-skippy-validate-sampling-across-all-execution-bounda.patch new file mode 100644 index 0000000000..15761723c8 --- /dev/null +++ b/third_party/llama.cpp/patches/0028-skippy-validate-sampling-across-all-execution-bounda.patch @@ -0,0 +1,303 @@ +From 2e57a2a5ec0bc3e7b5f1d633e5f541e67919b5ed Mon Sep 17 00:00:00 2001 +From: Mesh-LLM CI +Date: Sat, 12 Sep 2026 18:24:13 -0700 +Subject: [PATCH] skippy: validate sampling across all execution boundaries + +--- + src/skippy/execution-batch.cpp | 15 ++++++++--- + src/skippy/execution-single.cpp | 36 +++++++++++++++++-------- + src/skippy/sampling_internal.h | 3 +++ + src/skippy/sampling_vocab.cpp | 11 ++++++++ + src/skippy/session.cpp | 9 ++----- + src/skippy/speculative_decoding.cpp | 27 ++++++++++++++----- + src/skippy/verification.cpp | 9 +++---- + tests/test-skippy-sampling-suppress.cpp | 21 +++++++++++++++ + 8 files changed, 98 insertions(+), 33 deletions(-) + +diff --git a/src/skippy/execution-batch.cpp b/src/skippy/execution-batch.cpp +index b66e25499..bf7cc1ecb 100644 +--- a/src/skippy/execution-batch.cpp ++++ b/src/skippy/execution-batch.cpp +@@ -340,11 +340,14 @@ enum skippy_status skippy_iteration_batch_sampled( + // the sparse-row translation internally. + const int32_t logits_index = + static_cast(request_offsets[request_index + 1] - 1); +- out_sampled_request_indexes[sampled_output_index] = request_index; +- out_predicted_tokens[sampled_output_index] = skippy_sample_token_ith( ++ const skippy_status sample_status = skippy_store_sampled_token(skippy_sample_token_ith( + request.session, + request.sampling, +- logits_index); ++ logits_index), &out_predicted_tokens[sampled_output_index], out_error); ++ if (sample_status != SKIPPY_STATUS_OK) { ++ return sample_status; ++ } ++ out_sampled_request_indexes[sampled_output_index] = request_index; + sampled_output_index += 1; + } + +@@ -680,7 +683,11 @@ enum skippy_status skippy_decode_step_frame_batch_sampled( + } + if (request_logits) { + const skippy_sampling_config * request_sampling = sampling != nullptr ? sampling[i] : nullptr; +- out_predicted_tokens[i] = skippy_sample_token_ith(session, request_sampling, i); ++ const skippy_status sample_status = skippy_store_sampled_token( ++ skippy_sample_token_ith(session, request_sampling, i), &out_predicted_tokens[i], out_error); ++ if (sample_status != SKIPPY_STATUS_OK) { ++ return sample_status; ++ } + } else { + out_predicted_tokens[i] = -1; + } +diff --git a/src/skippy/execution-single.cpp b/src/skippy/execution-single.cpp +index 357222c13..967a6b1ed 100644 +--- a/src/skippy/execution-single.cpp ++++ b/src/skippy/execution-single.cpp +@@ -119,7 +119,8 @@ enum skippy_status skippy_decode_step_sampled( + } + + if (out_predicted_token != nullptr) { +- *out_predicted_token = skippy_sample_token(session, sampling); ++ return skippy_store_sampled_token( ++ skippy_sample_token(session, sampling), out_predicted_token, out_error); + } + + return skippy_success(out_error); +@@ -233,7 +234,11 @@ enum skippy_status skippy_decode_batch_sampled( + session->n_past += 1; + skippy_record_tokens(session, &token_ids[i], 1); + const skippy_sampling_config * request_sampling = sampling != nullptr ? sampling[i] : nullptr; +- out_predicted_tokens[i] = skippy_sample_token_ith(session, request_sampling, i); ++ const skippy_status sample_status = skippy_store_sampled_token( ++ skippy_sample_token_ith(session, request_sampling, i), &out_predicted_tokens[i], out_error); ++ if (sample_status != SKIPPY_STATUS_OK) { ++ return sample_status; ++ } + } + + return skippy_success(out_error); +@@ -281,12 +286,11 @@ enum skippy_status skippy_verify_tokens( + if (status == SKIPPY_STATUS_OK) { + const int32_t n_tokens = static_cast(token_count); + for (int32_t i = 0; i < n_tokens; ++i) { +- output_tokens[i] = skippy_greedy_sample_ith(session, i); +- if (output_tokens[i] == LLAMA_TOKEN_NULL) { ++ status = skippy_store_sampled_token( ++ skippy_greedy_sample_ith(session, i), &output_tokens[i], out_error); ++ if (status != SKIPPY_STATUS_OK) { + *out_token_count = 0; +- skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, +- "verification produced no eligible token after vocabulary suppression"); +- return SKIPPY_STATUS_RUNTIME_ERROR; ++ return status; + } + } + } +@@ -370,9 +374,12 @@ static enum skippy_status skippy_prefill_chunk_frame_impl( + return status; + } + +- if (out_predicted_token != nullptr) { +- *out_predicted_token = session->stage_model->config.include_output ? +- skippy_sample_token(session, sampling) : -1; ++ if (out_predicted_token != nullptr && session->stage_model->config.include_output) { ++ status = skippy_store_sampled_token( ++ skippy_sample_token(session, sampling), out_predicted_token, out_error); ++ if (status != SKIPPY_STATUS_OK) { ++ return status; ++ } + } + + return skippy_copy_output_activation_frame( +@@ -564,7 +571,14 @@ enum skippy_status skippy_decode_step_frame_sampled( + } + + if (out_predicted_token != nullptr) { +- *out_predicted_token = session->stage_model->config.include_output ? skippy_sample_token(session, sampling) : -1; ++ *out_predicted_token = -1; ++ if (session->stage_model->config.include_output) { ++ status = skippy_store_sampled_token( ++ skippy_sample_token(session, sampling), out_predicted_token, out_error); ++ if (status != SKIPPY_STATUS_OK) { ++ return status; ++ } ++ } + } + + return skippy_copy_output_activation_frame( +diff --git a/src/skippy/sampling_internal.h b/src/skippy/sampling_internal.h +index 4e60b256c..0c4e54724 100644 +--- a/src/skippy/sampling_internal.h ++++ b/src/skippy/sampling_internal.h +@@ -39,6 +39,9 @@ llama_token skippy_greedy_sample_allowed_logits( + int32_t suppress_token_count, + bool ignore_eog); + llama_token skippy_greedy_sample_ith(skippy_session * session, int32_t index); ++// Keep exhausted sampling out of public outputs and subsequent decode inputs. ++enum skippy_status skippy_store_sampled_token( ++ llama_token token, llama_token * output, skippy_error ** out_error); + llama_sampler * skippy_build_sampling_chain(skippy_session * session, const skippy_sampling_config * sampling); + std::vector skippy_add_vocab_suppress_token_biases( + std::vector biases, +diff --git a/src/skippy/sampling_vocab.cpp b/src/skippy/sampling_vocab.cpp +index 1242ed105..baf83db44 100644 +--- a/src/skippy/sampling_vocab.cpp ++++ b/src/skippy/sampling_vocab.cpp +@@ -1,4 +1,5 @@ + #include "skippy/sampling_internal.h" ++#include "skippy/errors.h" + + #include "llama.h" + +@@ -8,6 +9,16 @@ + #include + #include + ++enum skippy_status skippy_store_sampled_token( ++ llama_token token, llama_token * output, skippy_error ** out_error) { ++ if (token < 0) { ++ skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, "sampling produced no allowed token"); ++ return SKIPPY_STATUS_RUNTIME_ERROR; ++ } ++ *output = token; ++ return skippy_success(out_error); ++} ++ + std::vector skippy_add_vocab_suppress_token_biases( + std::vector biases, + const llama_token * suppress_tokens, +diff --git a/src/skippy/session.cpp b/src/skippy/session.cpp +index 50af78cb8..51d873866 100644 +--- a/src/skippy/session.cpp ++++ b/src/skippy/session.cpp +@@ -259,13 +259,8 @@ enum skippy_status skippy_session_sample_current( + skippy_set_error(out_error, SKIPPY_STATUS_INVALID_ARGUMENT, "session and out_predicted_token are required"); + return SKIPPY_STATUS_INVALID_ARGUMENT; + } +- *out_predicted_token = skippy_sample_token(session, sampling); +- if (*out_predicted_token == LLAMA_TOKEN_NULL) { +- skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, +- "sampling produced no eligible token after vocabulary suppression"); +- return SKIPPY_STATUS_RUNTIME_ERROR; +- } +- return skippy_success(out_error); ++ return skippy_store_sampled_token( ++ skippy_sample_token(session, sampling), out_predicted_token, out_error); + } + + enum skippy_status skippy_session_configure_chat_sampling( +diff --git a/src/skippy/speculative_decoding.cpp b/src/skippy/speculative_decoding.cpp +index b28ef9096..321852e87 100644 +--- a/src/skippy/speculative_decoding.cpp ++++ b/src/skippy/speculative_decoding.cpp +@@ -295,8 +295,16 @@ enum skippy_status skippy_mtp_sync_target_inputs( + for (int32_t out_i = 0; out_i + 1 < n_decode; ++out_i) { + batch.token[out_i] = batch.token[out_i + 1]; + } +- batch.token[n_decode - 1] = skippy_greedy_sample_context( +- llama_get_model(mtp_ctx), mtp_ctx, -1); ++ const skippy_status sample_status = skippy_store_sampled_token( ++ skippy_greedy_sample_context(llama_get_model(mtp_ctx), mtp_ctx, -1), ++ &batch.token[n_decode - 1], out_error); ++ if (sample_status != SKIPPY_STATUS_OK) { ++ skippy_mtp_clear_session_state(session); ++ std::free(batch.token); ++ batch.token = nullptr; ++ llama_batch_free(batch); ++ return sample_status; ++ } + } + if (batch.token != nullptr) { + std::free(batch.token); +@@ -460,8 +468,13 @@ enum skippy_status skippy_mtp_propose_next( + } + + token = skippy_greedy_sample_context(llama_get_model(mtp_ctx), mtp_ctx, -1); +- if (token == LLAMA_TOKEN_NULL) { +- break; ++ const skippy_status sample_status = skippy_store_sampled_token(token, &token, out_error); ++ if (sample_status != SKIPPY_STATUS_OK) { ++ skippy_mtp_clear_session_state(session); ++ std::free(batch.token); ++ batch.token = nullptr; ++ llama_batch_free(batch); ++ return sample_status; + } + if (out_mtp_draft != nullptr) { + out_mtp_draft->token_ids[token_count] = token; +@@ -523,8 +536,10 @@ enum skippy_status skippy_mtp_propose_next( + } + + token = skippy_greedy_sample_context(llama_get_model(mtp_ctx), mtp_ctx, -1); +- if (token == LLAMA_TOKEN_NULL) { +- break; ++ const skippy_status sample_status = skippy_store_sampled_token(token, &token, out_error); ++ if (sample_status != SKIPPY_STATUS_OK) { ++ skippy_mtp_clear_session_state(session); ++ return sample_status; + } + if (out_mtp_draft != nullptr) { + out_mtp_draft->token_ids[token_count] = token; +diff --git a/src/skippy/verification.cpp b/src/skippy/verification.cpp +index c14e7a0a3..09a8a2012 100644 +--- a/src/skippy/verification.cpp ++++ b/src/skippy/verification.cpp +@@ -229,12 +229,11 @@ enum skippy_status skippy_verify_tokens_frame_sampled( + for (size_t i = 0; i < token_count; ++i) { + skippy_record_tokens(session, &token_ids[i], 1); + const int32_t logits_index = static_cast(i); +- output_tokens[i] = skippy_sample_token_ith(session, sampling, logits_index); +- if (output_tokens[i] == LLAMA_TOKEN_NULL) { ++ status = skippy_store_sampled_token( ++ skippy_sample_token_ith(session, sampling, logits_index), &output_tokens[i], out_error); ++ if (status != SKIPPY_STATUS_OK) { + *out_token_count = 0; +- skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, +- "sampled verification produced no eligible token after vocabulary suppression"); +- return SKIPPY_STATUS_RUNTIME_ERROR; ++ return status; + } + if (stop_after_first_mismatch && i + 1 < token_count && output_tokens[i] != token_ids[i + 1]) { + // Later rows are conditioned on a rejected token and are not +diff --git a/tests/test-skippy-sampling-suppress.cpp b/tests/test-skippy-sampling-suppress.cpp +index 4cf702bef..3f797cbe8 100644 +--- a/tests/test-skippy-sampling-suppress.cpp ++++ b/tests/test-skippy-sampling-suppress.cpp +@@ -4,6 +4,7 @@ + + #include + #include ++#include + #include + + static bool suppressed(const llama_logit_bias & bias) { +@@ -60,6 +61,26 @@ int main() { + return 4; + } + ++ llama_token output = 42; ++ skippy_error * error = nullptr; ++ const llama_token exhausted = skippy_greedy_sample_allowed_logits( ++ logits, nullptr, 5, all_tokens, 5, false); ++ if (skippy_store_sampled_token(exhausted, &output, &error) != SKIPPY_STATUS_RUNTIME_ERROR || ++ output != 42 || error == nullptr || ++ std::strstr(error->message, "no allowed token") == nullptr) { ++ std::fprintf(stderr, "exhausted sampling leaked into the output buffer\n"); ++ skippy_error_free(error); ++ return 6; ++ } ++ skippy_error_free(error); ++ error = nullptr; ++ if (skippy_store_sampled_token(0, &output, &error) != SKIPPY_STATUS_OK || ++ output != 0 || error != nullptr) { ++ std::fprintf(stderr, "valid token zero was rejected\n"); ++ skippy_error_free(error); ++ return 7; ++ } ++ + merged = skippy_add_vocab_suppress_token_biases(caller_biases, nullptr, 0); + if (merged.size() != caller_biases.size()) { + std::fprintf(stderr, "models without suppress tokens lost caller logit biases\n"); +-- +2.54.0 (Apple Git-157) + From 551c8d92c7c8fdfba27ea352cbdcc7aebe08d6b9 Mon Sep 17 00:00:00 2001 From: IvGolovach <20299097+IvGolovach@users.noreply.github.com> Date: Sat, 12 Sep 2026 22:01:19 -0700 Subject: [PATCH 12/18] test: require integer indexes in embedding certification --- scripts/ci-openai-embeddings-smoke.py | 2 +- scripts/ci-openai-workload-smoke.py | 3 ++- scripts/tests/test_ci_openai_embeddings_smoke.py | 6 +++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/ci-openai-embeddings-smoke.py b/scripts/ci-openai-embeddings-smoke.py index 769cd63380..ebf802b25a 100755 --- a/scripts/ci-openai-embeddings-smoke.py +++ b/scripts/ci-openai-embeddings-smoke.py @@ -62,7 +62,7 @@ def main() -> None: if len(encoded.data) != 1: raise RuntimeError("base64 embeddings response has the wrong batch size") item = encoded.data[0] - if item.object != "embedding" or item.index != 0: + if item.object != "embedding" or type(item.index) is not int or item.index != 0: raise RuntimeError("base64 embeddings response has invalid item metadata") payload = item.embedding if not isinstance(payload, str): diff --git a/scripts/ci-openai-workload-smoke.py b/scripts/ci-openai-workload-smoke.py index 4fcaed8bdb..e7c623b280 100644 --- a/scripts/ci-openai-workload-smoke.py +++ b/scripts/ci-openai-workload-smoke.py @@ -125,7 +125,8 @@ def smoke_embedding(base_url: str, model: str) -> None: if not isinstance(rows, list) or len(rows) != 1: raise RuntimeError("base64 embedding response has the wrong batch size") item = rows[0] - if not isinstance(item, dict) or item.get("object") != "embedding" or item.get("index") != 0: + if (not isinstance(item, dict) or item.get("object") != "embedding" + or type(item.get("index")) is not int or item["index"] != 0): raise RuntimeError("base64 embedding response has invalid item metadata") payload = item.get("embedding") if not isinstance(payload, str): diff --git a/scripts/tests/test_ci_openai_embeddings_smoke.py b/scripts/tests/test_ci_openai_embeddings_smoke.py index f16ef2f711..7cf06e705c 100644 --- a/scripts/tests/test_ci_openai_embeddings_smoke.py +++ b/scripts/tests/test_ci_openai_embeddings_smoke.py @@ -81,7 +81,8 @@ def test_response_metadata_must_match_request(self) -> None: def test_item_metadata_must_match_single_input(self) -> None: """Reject missing, negative and surplus indexes and invalid item objects.""" for field, value in (("object", "list"), ("object", None), - ("index", -1), ("index", 1), ("index", None)): + ("index", -1), ("index", 1), ("index", None), + ("index", False), ("index", 0.0), ("index", "0")): with self.subTest(field=field, value=value): response = self.encoded_response() setattr(response.data[0], field, value) @@ -122,6 +123,9 @@ def test_base64_cardinality_and_metadata_are_required(self) -> None: self.run_smoke(good) for changed in ({"data": []}, {"data": [item, item]}, {"data": None}, {"data": [None]}, {"data": [{**item, "index": 1}]}, + {"data": [{**item, "index": False}]}, + {"data": [{**item, "index": 0.0}]}, + {"data": [{**item, "index": "0"}]}, {"data": [{**item, "object": None}]}, {"model": "another-model"}, {"object": "embedding"}): with self.subTest(changed=changed), self.assertRaises(RuntimeError): From ca0ef0f79fec4f3ae1dc8a15082f6c529029cd43 Mon Sep 17 00:00:00 2001 From: IvGolovach <20299097+IvGolovach@users.noreply.github.com> Date: Sat, 12 Sep 2026 23:44:02 -0700 Subject: [PATCH 13/18] fix: reject invalid workload routes and certification evidence Reject oversized multipart destinations and preserve the authority of modern descriptors when their invalid entries are discarded. Retain legacy inference only for peers that omit descriptors. Require input-dependent embeddings, meaningful rerank ordering, and float/base64 value parity in workload certification. Cover malformed outputs and boundary cases with regression tests. Authorize the existing Linux runtime-event fixture for PR and main retrieval without expanding family-certification schedules. Regenerate the model manifests and document the new workload contracts. --- .../manage-ci/references/current-inventory.md | 3 + ci/ci.md | 3 + .../manifests/competitive-benchmark.json | 2 +- .../manifests/hf-download-smoke.json | 2 +- .../manifests/openai-smoke.json | 2 +- .../manifests/product-integration-smoke.json | 2 +- .../manifests/product-smoke.json | 2 +- ci/model-artifacts/manifests/radix-cache.json | 2 +- .../manifests/safetensors-runtime-smoke.json | 2 +- .../manifests/scripted-binary-smoke.json | 2 +- ci/model-artifacts/manifests/sdk-smoke.json | 2 +- .../manifests/skippy-ci-smoke.json | 4 +- .../manifests/skippy-correctness.json | 2 +- .../manifests/skippy-parity.json | 2 +- ci/model-artifacts/registry.json | 2 + .../src/inference/skippy/mod.rs | 6 + .../src/mesh/model_identity.rs | 2 + .../src/network/openai/ingress.rs | 2 + .../ingress_tests/request_object_cleanup.rs | 2 + .../src/network/openai/request_parse.rs | 1 + .../openai/request_parse/audio_multipart.rs | 5 +- .../request_parse/audio_multipart_tests.rs | 199 ++++++++++++++++++ .../openai/request_parse/body_rewrite.rs | 3 + .../src/network/openai/request_parse_tests.rs | 154 +------------- .../network/openai/transport_tests/routing.rs | 2 + .../src/protocol/convert.rs | 98 +-------- .../protocol/convert/served_descriptors.rs | 191 +++++++++++++++++ crates/openai-frontend/src/audio.rs | 2 + crates/openai-frontend/src/embeddings.rs | 3 + .../openai-frontend/src/guardrails/compact.rs | 5 + crates/openai-frontend/src/guardrails/mod.rs | 5 + crates/openai-frontend/src/hooks.rs | 5 + crates/openai-frontend/src/rerank.rs | 1 + crates/openai-frontend/src/router_tests.rs | 5 + crates/skippy-ffi/build.rs | 2 + crates/skippy-ffi/src/abi.rs | 1 + crates/skippy-runtime/src/media.rs | 4 + .../src/media/speech_session_tests.rs | 3 + crates/skippy-runtime/src/native.rs | 3 + crates/skippy-server/src/frontend/backend.rs | 5 + .../src/frontend/backend/tests.rs | 3 + .../generation_flow/encoder_decoder.rs | 1 + .../src/frontend/tests/non_chat.rs | 75 ++++++- .../src/frontend/tests/support.rs | 1 + crates/skippy-server/src/http.rs | 2 + scripts/ci-openai-embeddings-smoke.py | 3 + scripts/ci-openai-workload-smoke.py | 16 +- scripts/ci-workload-monolithic-oracle.py | 3 + scripts/generate-ocr-oracle-fixture.py | 3 + scripts/llama-oracle-source.py | 4 + scripts/skippy-ocr-asr-oracle.py | 7 + .../test_check_skippy_workload_candidate.py | 5 + .../tests/test_ci_openai_embeddings_smoke.py | 31 +++ scripts/tests/test_llama_oracle_source.py | 2 + .../test_llama_upstream_canary_contract.py | 3 + scripts/tests/test_plan_family_battery.py | 11 + .../test_runtime_events_model_cadence.py | 79 +++++++ scripts/tests/test_skippy_static_link.py | 10 + scripts/tests/test_skippy_tts_oracle.py | 1 + 59 files changed, 747 insertions(+), 256 deletions(-) create mode 100644 crates/mesh-llm-host-runtime/src/network/openai/request_parse/audio_multipart_tests.rs create mode 100644 crates/mesh-llm-host-runtime/src/protocol/convert/served_descriptors.rs create mode 100644 scripts/tests/test_runtime_events_model_cadence.py diff --git a/.agents/skills/manage-ci/references/current-inventory.md b/.agents/skills/manage-ci/references/current-inventory.md index ce198d9b68..3d7d2d99f1 100644 --- a/.agents/skills/manage-ci/references/current-inventory.md +++ b/.agents/skills/manage-ci/references/current-inventory.md @@ -595,6 +595,9 @@ fail-open policy. `model_artifact_id` selects one artifact from a multi-artifact manifest, and reaches both the resolve and the verify call so verification cannot check a different file than the one downloaded. + The Linux native runtime-event gate selects `family-qwen3-dense` from the + Skippy smoke manifest; the registry permits `pull-request` and `main` + restoration while retaining its independent family-certification cadences. - `restore-smoke-inputs`: product extraction for consumers; delegates model restoration to `restore-test-model` rather than carrying a second copy of that sequence. diff --git a/ci/ci.md b/ci/ci.md index f5d8611ec6..c4c61e2215 100644 --- a/ci/ci.md +++ b/ci/ci.md @@ -399,6 +399,9 @@ runtime producers are not duplicated. freshly built native runtime. CPU only — the reporter is backend-independent, so another backend would buy a duplicate of the same evidence. + Its exact `family-qwen3-dense` fixture is registry-authorized for both + `pull-request` and `main` restoration. Those retrieval cadences do not expand + the separate persistent-runner family-certification schedule. - `ci-{linux,macos,windows}-product-slice.yml` — composition-only consumers that join only their matching immutable host and runtime artifacts. - `ci-platform-checks-slice.yml` — macOS portable/unit, Windows portable, and diff --git a/ci/model-artifacts/manifests/competitive-benchmark.json b/ci/model-artifacts/manifests/competitive-benchmark.json index d964cae23e..9da1a598b4 100644 --- a/ci/model-artifacts/manifests/competitive-benchmark.json +++ b/ci/model-artifacts/manifests/competitive-benchmark.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "competitive-benchmark", - "registry_sha256": "70665c803b4d65dc6e2373314dce4b928698c3836774b52504655f8fac778614", + "registry_sha256": "e320721a799f80117466bda63b3ad4496bfb316e6c34f5f5f0ee29d4b8a32466", "artifacts": [ { "id": "family-llama", diff --git a/ci/model-artifacts/manifests/hf-download-smoke.json b/ci/model-artifacts/manifests/hf-download-smoke.json index 25163fedaf..a0bd360bd6 100644 --- a/ci/model-artifacts/manifests/hf-download-smoke.json +++ b/ci/model-artifacts/manifests/hf-download-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "hf-download-smoke", - "registry_sha256": "70665c803b4d65dc6e2373314dce4b928698c3836774b52504655f8fac778614", + "registry_sha256": "e320721a799f80117466bda63b3ad4496bfb316e6c34f5f5f0ee29d4b8a32466", "artifacts": [ { "id": "smollm2-q4-download", diff --git a/ci/model-artifacts/manifests/openai-smoke.json b/ci/model-artifacts/manifests/openai-smoke.json index 22c3a2e367..ba31e122c9 100644 --- a/ci/model-artifacts/manifests/openai-smoke.json +++ b/ci/model-artifacts/manifests/openai-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "openai-smoke", - "registry_sha256": "70665c803b4d65dc6e2373314dce4b928698c3836774b52504655f8fac778614", + "registry_sha256": "e320721a799f80117466bda63b3ad4496bfb316e6c34f5f5f0ee29d4b8a32466", "artifacts": [ { "id": "smollm2-q4-download", diff --git a/ci/model-artifacts/manifests/product-integration-smoke.json b/ci/model-artifacts/manifests/product-integration-smoke.json index 06d1d0073b..fab95e5f59 100644 --- a/ci/model-artifacts/manifests/product-integration-smoke.json +++ b/ci/model-artifacts/manifests/product-integration-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "product-integration-smoke", - "registry_sha256": "70665c803b4d65dc6e2373314dce4b928698c3836774b52504655f8fac778614", + "registry_sha256": "e320721a799f80117466bda63b3ad4496bfb316e6c34f5f5f0ee29d4b8a32466", "artifacts": [ { "id": "family-granite-hybrid", diff --git a/ci/model-artifacts/manifests/product-smoke.json b/ci/model-artifacts/manifests/product-smoke.json index 9bdc932371..32ba2b0178 100644 --- a/ci/model-artifacts/manifests/product-smoke.json +++ b/ci/model-artifacts/manifests/product-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "product-smoke", - "registry_sha256": "70665c803b4d65dc6e2373314dce4b928698c3836774b52504655f8fac778614", + "registry_sha256": "e320721a799f80117466bda63b3ad4496bfb316e6c34f5f5f0ee29d4b8a32466", "artifacts": [ { "id": "smollm2-q8-inference", diff --git a/ci/model-artifacts/manifests/radix-cache.json b/ci/model-artifacts/manifests/radix-cache.json index a9f0a6c4ac..12824cb8a4 100644 --- a/ci/model-artifacts/manifests/radix-cache.json +++ b/ci/model-artifacts/manifests/radix-cache.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "radix-cache", - "registry_sha256": "70665c803b4d65dc6e2373314dce4b928698c3836774b52504655f8fac778614", + "registry_sha256": "e320721a799f80117466bda63b3ad4496bfb316e6c34f5f5f0ee29d4b8a32466", "artifacts": [ { "id": "family-deepseek2", diff --git a/ci/model-artifacts/manifests/safetensors-runtime-smoke.json b/ci/model-artifacts/manifests/safetensors-runtime-smoke.json index f59e8fd571..6169605665 100644 --- a/ci/model-artifacts/manifests/safetensors-runtime-smoke.json +++ b/ci/model-artifacts/manifests/safetensors-runtime-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "safetensors-runtime-smoke", - "registry_sha256": "70665c803b4d65dc6e2373314dce4b928698c3836774b52504655f8fac778614", + "registry_sha256": "e320721a799f80117466bda63b3ad4496bfb316e6c34f5f5f0ee29d4b8a32466", "artifacts": [ { "id": "smollm2-safetensors", diff --git a/ci/model-artifacts/manifests/scripted-binary-smoke.json b/ci/model-artifacts/manifests/scripted-binary-smoke.json index 3bfbcb8639..2d6fb06fac 100644 --- a/ci/model-artifacts/manifests/scripted-binary-smoke.json +++ b/ci/model-artifacts/manifests/scripted-binary-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "scripted-binary-smoke", - "registry_sha256": "70665c803b4d65dc6e2373314dce4b928698c3836774b52504655f8fac778614", + "registry_sha256": "e320721a799f80117466bda63b3ad4496bfb316e6c34f5f5f0ee29d4b8a32466", "artifacts": [ { "id": "smollm2-q8-inference", diff --git a/ci/model-artifacts/manifests/sdk-smoke.json b/ci/model-artifacts/manifests/sdk-smoke.json index 2e0afeddfc..6972ecb9c0 100644 --- a/ci/model-artifacts/manifests/sdk-smoke.json +++ b/ci/model-artifacts/manifests/sdk-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "sdk-smoke", - "registry_sha256": "70665c803b4d65dc6e2373314dce4b928698c3836774b52504655f8fac778614", + "registry_sha256": "e320721a799f80117466bda63b3ad4496bfb316e6c34f5f5f0ee29d4b8a32466", "artifacts": [ { "id": "smollm2-q8-inference", diff --git a/ci/model-artifacts/manifests/skippy-ci-smoke.json b/ci/model-artifacts/manifests/skippy-ci-smoke.json index c3ef89206e..ad9d86e9d3 100644 --- a/ci/model-artifacts/manifests/skippy-ci-smoke.json +++ b/ci/model-artifacts/manifests/skippy-ci-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "skippy-ci-smoke", - "registry_sha256": "70665c803b4d65dc6e2373314dce4b928698c3836774b52504655f8fac778614", + "registry_sha256": "e320721a799f80117466bda63b3ad4496bfb316e6c34f5f5f0ee29d4b8a32466", "artifacts": [ { "id": "family-qwen3-dense", @@ -17,6 +17,8 @@ "skippy-ci-smoke" ], "cadences": [ + "pull-request", + "main", "llama-bump", "manual-full", "nightly", diff --git a/ci/model-artifacts/manifests/skippy-correctness.json b/ci/model-artifacts/manifests/skippy-correctness.json index bc06d29a91..9b9628676b 100644 --- a/ci/model-artifacts/manifests/skippy-correctness.json +++ b/ci/model-artifacts/manifests/skippy-correctness.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "skippy-correctness", - "registry_sha256": "70665c803b4d65dc6e2373314dce4b928698c3836774b52504655f8fac778614", + "registry_sha256": "e320721a799f80117466bda63b3ad4496bfb316e6c34f5f5f0ee29d4b8a32466", "artifacts": [ { "id": "qwen3-q8-correctness", diff --git a/ci/model-artifacts/manifests/skippy-parity.json b/ci/model-artifacts/manifests/skippy-parity.json index 6c966872b2..1fabe71be6 100644 --- a/ci/model-artifacts/manifests/skippy-parity.json +++ b/ci/model-artifacts/manifests/skippy-parity.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "skippy-parity", - "registry_sha256": "70665c803b4d65dc6e2373314dce4b928698c3836774b52504655f8fac778614", + "registry_sha256": "e320721a799f80117466bda63b3ad4496bfb316e6c34f5f5f0ee29d4b8a32466", "artifacts": [ { "id": "family-deepseek2", diff --git a/ci/model-artifacts/registry.json b/ci/model-artifacts/registry.json index 85f8a109ce..c12ceb341c 100644 --- a/ci/model-artifacts/registry.json +++ b/ci/model-artifacts/registry.json @@ -100,6 +100,8 @@ "skippy-ci-smoke" ], "cadences": [ + "pull-request", + "main", "llama-bump", "manual-full", "nightly", diff --git a/crates/mesh-llm-host-runtime/src/inference/skippy/mod.rs b/crates/mesh-llm-host-runtime/src/inference/skippy/mod.rs index 8d28b98d0d..038670dc04 100644 --- a/crates/mesh-llm-host-runtime/src/inference/skippy/mod.rs +++ b/crates/mesh-llm-host-runtime/src/inference/skippy/mod.rs @@ -658,6 +658,7 @@ impl SkippyModelHandle { self.runtime.output_activation_boundary() } + /// Classify the loaded native runtime, including speech-capable projectors. pub(crate) fn workload_class(&self) -> Result { if self.runtime.supports_speech_synthesis() { return Ok(crate::mesh::ModelWorkloadClass::SpeechSynthesis); @@ -1248,6 +1249,7 @@ impl OpenAiBackend for SkippyModelHandle { self.backend.completion_stream(request, context).await } + /// Forward embeddings and request context without chat processing. async fn embeddings( &self, request: EmbeddingsRequest, @@ -1256,6 +1258,7 @@ impl OpenAiBackend for SkippyModelHandle { self.backend.embeddings(request, context).await } + /// Forward reranking and request context without chat processing. async fn rerank( &self, request: RerankRequest, @@ -1264,6 +1267,7 @@ impl OpenAiBackend for SkippyModelHandle { self.backend.rerank(request, context).await } + /// Forward speech generation and request context unchanged. async fn audio_speech( &self, request: AudioSpeechRequest, @@ -1272,6 +1276,7 @@ impl OpenAiBackend for SkippyModelHandle { self.backend.audio_speech(request, context).await } + /// Forward multipart transcription and request context unchanged. async fn audio_transcription( &self, request: AudioTranscriptionRequest, @@ -1280,6 +1285,7 @@ impl OpenAiBackend for SkippyModelHandle { self.backend.audio_transcription(request, context).await } + /// Forward multipart translation and request context unchanged. async fn audio_translation( &self, request: AudioTranscriptionRequest, diff --git a/crates/mesh-llm-host-runtime/src/mesh/model_identity.rs b/crates/mesh-llm-host-runtime/src/mesh/model_identity.rs index 36f31ac09f..014475c6b1 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/model_identity.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/model_identity.rs @@ -211,6 +211,7 @@ pub(crate) fn identity_hash_for(input: &str) -> String { hex::encode(hasher.finalize()) } +/// Expose only remotely resolvable model identities as public catalog IDs. pub(crate) fn public_model_id_from_identity(identity: &ServedModelIdentity) -> Option { match identity.source_kind { ModelSourceKind::HuggingFace => identity @@ -240,6 +241,7 @@ pub(crate) fn public_model_id_from_identity(identity: &ServedModelIdentity) -> O } } +/// Normalize demand references without converting local paths into remote model identities. pub(crate) fn canonical_demand_model_ref(model: &str) -> String { if let Ok(model_ref) = model_ref::ModelRef::parse(model) { return model_ref.display_id(); diff --git a/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs b/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs index b9beced35f..6cca190ad8 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs @@ -880,6 +880,7 @@ async fn prepare_auto_route_decision( } } +/// Return a path-specific unsupported-workload response and record the rejection. async fn send_workload_unsupported( tcp_stream: ClientStream, workload: mesh::ModelWorkloadClass, @@ -913,6 +914,7 @@ async fn send_media_unsupported( ) } +/// Release request-scoped media objects before reporting an automatic-routing rejection. async fn send_auto_route_rejection( tcp_stream: ClientStream, rejection: AutoRouteRejection, diff --git a/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/request_object_cleanup.rs b/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/request_object_cleanup.rs index d205312c7a..10471aea97 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/request_object_cleanup.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/request_object_cleanup.rs @@ -10,6 +10,7 @@ use tokio::net::{TcpListener, TcpStream}; struct CompletionRecorder(Mutex>); impl PluginRpcBridge for CompletionRecorder { + /// Record blob completion requests so rejection tests can verify object cleanup. fn handle_request( &self, plugin_name: String, @@ -38,6 +39,7 @@ impl PluginRpcBridge for CompletionRecorder { }) } + /// Ignore notifications in the request-object cleanup test double. fn handle_notification(&self, _: String, _: String, _: String) -> BridgeFuture<()> { Box::pin(async {}) } diff --git a/crates/mesh-llm-host-runtime/src/network/openai/request_parse.rs b/crates/mesh-llm-host-runtime/src/network/openai/request_parse.rs index c3ebb98c81..63e857af49 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/request_parse.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/request_parse.rs @@ -567,6 +567,7 @@ fn body_limits_for_path(path: &str, default: HttpReadLimits) -> HttpReadLimits { } } +/// Identify multipart audio endpoints before attempting JSON parsing. fn is_audio_upload_path(path: &str) -> bool { matches!( path.split('?').next().unwrap_or(path), diff --git a/crates/mesh-llm-host-runtime/src/network/openai/request_parse/audio_multipart.rs b/crates/mesh-llm-host-runtime/src/network/openai/request_parse/audio_multipart.rs index ef32e3fb61..305a8ca6d8 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/request_parse/audio_multipart.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/request_parse/audio_multipart.rs @@ -183,5 +183,8 @@ pub(super) fn multipart_model_field(content_type: &str, body: &[u8]) -> Result 256 { + bail!("multipart model field exceeds the 256-byte limit"); + } + Ok((!value.is_empty()).then(|| value.to_string())) } diff --git a/crates/mesh-llm-host-runtime/src/network/openai/request_parse/audio_multipart_tests.rs b/crates/mesh-llm-host-runtime/src/network/openai/request_parse/audio_multipart_tests.rs new file mode 100644 index 0000000000..eb84731f4b --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/network/openai/request_parse/audio_multipart_tests.rs @@ -0,0 +1,199 @@ +//! Multipart routing and byte-preserving request regression tests. + +use super::super::audio_multipart::{multipart_boundary, multipart_part_is_model}; +use super::*; + +#[tokio::test] +/// Model routing must never decode or rewrite the uploaded audio bytes. +async fn multipart_model_is_parsed_and_rewritten_without_touching_file_bytes() { + const BOUNDARY: &str = "mesh-audio-boundary"; + // A boundary prefix inside binary content is not a multipart delimiter. + let file_bytes = [ + 0_u8, 255, 13, 10, b'-', b'-', b'm', b'e', b's', b'h', b'-', b'a', b'u', b'd', b'i', b'o', + b'-', b'b', b'o', b'u', b'n', b'd', b'a', b'r', b'y', b'X', 13, 10, 1, 2, 3, 128, + ]; + let mut body = format!( + "--{BOUNDARY}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"voice; name=model\"\r\nContent-Type: audio/wav\r\n\r\n" + ) + .into_bytes(); + body.extend_from_slice(&file_bytes); + body.extend_from_slice( + format!( + "\r\n--{BOUNDARY}\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\nauto\r\n--{BOUNDARY}--\r\n" + ) + .as_bytes(), + ); + let headers = format!( + "POST /v1/audio/transcriptions HTTP/1.1\r\nHost: localhost\r\nContent-Type: multipart/form-data; boundary=\"{BOUNDARY}\"\r\nContent-Length: {}\r\n\r\n", + body.len() + ) + .into_bytes(); + + let mut request = read_request_from_parts(vec![headers, body]).await; + assert_eq!(request.model_name.as_deref(), Some("auto")); + + rewrite_model_field(&mut request, "whisper-local"); + + assert_eq!(request.model_name.as_deref(), Some("whisper-local")); + assert!( + request + .raw + .windows(file_bytes.len()) + .any(|window| window == file_bytes) + ); + let header_end = request + .raw + .windows(4) + .position(|window| window == b"\r\n\r\n") + .unwrap() + + 4; + let content_type = format!("multipart/form-data; boundary={BOUNDARY}"); + assert_eq!( + multipart_model_field(&content_type, &request.raw[header_end..]) + .unwrap() + .as_deref(), + Some("whisper-local") + ); + let declared = std::str::from_utf8(&request.raw[..header_end]) + .unwrap() + .lines() + .find_map(|line| { + line.split_once(':') + .filter(|(name, _)| name.eq_ignore_ascii_case("content-length")) + .and_then(|(_, value)| value.trim().parse::().ok()) + }) + .unwrap(); + assert_eq!(declared, request.raw.len() - header_end); + assert_eq!(declared, request.body_len_bytes); +} + +#[test] +/// Reject ambiguous framing and unbounded model identifiers before route selection. +fn multipart_parser_rejects_invalid_boundaries_and_oversized_model_values() { + assert!(multipart_boundary("multipart/form-data; boundary=bad space").is_none()); + assert!(multipart_boundary("application/json; boundary=mesh").is_none()); + + let boundary = "mesh"; + let body = format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\n{}\r\n--{boundary}--\r\n", + "x".repeat(257) + ); + let error = multipart_model_field( + &format!("multipart/form-data; boundary={boundary}"), + body.as_bytes(), + ) + .unwrap_err(); + assert!(error.to_string().contains("256-byte limit")); +} + +/// The identifier ceiling counts UTF-8 bytes; absent and blank fields stay optional. +#[test] +fn multipart_model_limit_preserves_valid_and_missing_values() { + let content_type = "multipart/form-data; boundary=mesh"; + for value in ["x".repeat(256), "é".repeat(128), " auto ".to_string()] { + let body = format!( + "--mesh\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\n{value}\r\n--mesh--\r\n" + ); + assert_eq!( + multipart_model_field(content_type, body.as_bytes()).unwrap(), + Some(value.trim().to_string()) + ); + } + for body in [ + b"--mesh--\r\n".as_slice(), + b"--mesh\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\n \t\r\n--mesh--\r\n", + b"--mesh\r\nContent-Disposition: form-data; name=\"file\"; filename=\"audio.wav\"\r\n\r\nWAVE\r\n--mesh--\r\n", + ] { + assert_eq!(multipart_model_field(content_type, body).unwrap(), None); + } +} + +/// Both audio endpoints reject overlong explicit destinations instead of routing automatically. +#[tokio::test] +async fn oversized_multipart_model_is_rejected_before_audio_routing() { + for path in ["/v1/audio/transcriptions", "/v1/audio/translations"] { + for value in ["x".repeat(257), "é".repeat(129)] { + let body = format!( + "--mesh\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\n{value}\r\n--mesh--\r\n" + ); + let request = format!( + "POST {path} HTTP/1.1\r\nContent-Type: multipart/form-data; boundary=mesh\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ); + let (mut client, mut server) = tokio::io::duplex(request.len() + 1); + client.write_all(request.as_bytes()).await.unwrap(); + client.shutdown().await.unwrap(); + let error = read_http_request_with_plugin_manager_with_context(&mut server, None) + .await + .unwrap_err(); + assert_eq!(error.context().unwrap().client_path, path); + assert!(error.to_string().contains("256-byte limit")); + } + } +} + +#[tokio::test] +/// Two model fields cannot disagree about the destination of one upload. +async fn duplicate_multipart_model_is_rejected_before_audio_routing() { + let boundary = "mesh-audio-boundary"; + let body = format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\nauto\r\n\ + --{boundary}\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\nother-model\r\n\ + --{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"voice.wav\"\r\n\r\nWAVE\r\n\ + --{boundary}--\r\n" + ); + let content_type = format!("multipart/form-data; boundary={boundary}"); + assert!( + multipart_model_field(&content_type, body.as_bytes()) + .unwrap_err() + .to_string() + .contains("duplicate multipart model field") + ); + + for path in ["/v1/audio/transcriptions", "/v1/audio/translations"] { + let request = format!( + "POST {path} HTTP/1.1\r\nHost: localhost\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ); + let (mut client, mut server) = tokio::io::duplex(request.len() + 1); + client.write_all(request.as_bytes()).await.unwrap(); + client.shutdown().await.unwrap(); + let error = read_http_request_with_plugin_manager_with_context(&mut server, None) + .await + .unwrap_err(); + assert_eq!(error.context().unwrap().client_path, path); + assert!( + error + .to_string() + .contains("duplicate multipart model field") + ); + } +} + +#[test] +/// A boundary-like sequence inside payload data is not a valid multipart start. +fn multipart_model_scanner_rejects_non_initial_boundary() { + let body = b"binary--mesh\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\nauto\r\n--mesh--\r\n"; + let error = multipart_model_field("multipart/form-data; boundary=mesh", body).unwrap_err(); + assert!( + error + .to_string() + .contains("must start with its declared boundary") + ); +} + +#[test] +/// Quoted filenames cannot impersonate the disposition's model-field parameter. +fn multipart_disposition_ignores_name_like_text_inside_quoted_filename() { + assert!( + !multipart_part_is_model( + "Content-Disposition: form-data; name=\"file\"; filename=\"voice; name=model\"" + ) + .unwrap() + ); + assert!(multipart_part_is_model("Content-Disposition: form-data; name=\"model\"").unwrap()); + assert!( + multipart_part_is_model("Content-Disposition: form-data; name=\"file\"; name=\"model\"") + .is_err() + ); +} diff --git a/crates/mesh-llm-host-runtime/src/network/openai/request_parse/body_rewrite.rs b/crates/mesh-llm-host-runtime/src/network/openai/request_parse/body_rewrite.rs index f56b8e313b..a21c8682b5 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/request_parse/body_rewrite.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/request_parse/body_rewrite.rs @@ -13,6 +13,7 @@ struct BodyHeaders<'a> { } impl BodyHeaders<'_> { + /// Allow JSON rewriting only for an absent or application/json media type. fn permits_json(&self) -> bool { self.content_type.is_none_or(|value| { value.split(';').next().is_some_and(|media_type| { @@ -22,6 +23,7 @@ impl BodyHeaders<'_> { } } +/// Extract framing and media type from complete HTTP request headers. fn body_headers(raw: &[u8]) -> Option> { let mut headers = [httparse::EMPTY_HEADER; MAX_HEADERS]; let mut parsed = httparse::Request::new(&mut headers); @@ -100,6 +102,7 @@ pub fn inject_mesh_hooks_flag(raw: &mut Vec, enabled: bool) { replace_body(raw, end, &body); } +/// Replace request framing and parsed body state together after a JSON rewrite. fn rebuild_request_body( request: &mut BufferedHttpRequest, header_end: usize, diff --git a/crates/mesh-llm-host-runtime/src/network/openai/request_parse_tests.rs b/crates/mesh-llm-host-runtime/src/network/openai/request_parse_tests.rs index 9ddbb7126a..f8e1fb10e3 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/request_parse_tests.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/request_parse_tests.rs @@ -1,5 +1,6 @@ -use super::audio_multipart::{multipart_boundary, multipart_part_is_model}; use super::*; +#[path = "request_parse/audio_multipart_tests.rs"] +mod audio_multipart_tests; use tokio::io::AsyncWriteExt; use tokio::net::TcpListener; use tokio::net::TcpStream; @@ -828,157 +829,6 @@ fn test_rewrite_model_field_updates_body_and_content_length() { assert_eq!(declared, request.body_len_bytes); } -#[tokio::test] -/// Model routing must never decode or rewrite the uploaded audio bytes. -async fn multipart_model_is_parsed_and_rewritten_without_touching_file_bytes() { - const BOUNDARY: &str = "mesh-audio-boundary"; - // A boundary prefix inside binary content is not a multipart delimiter. - let file_bytes = [ - 0_u8, 255, 13, 10, b'-', b'-', b'm', b'e', b's', b'h', b'-', b'a', b'u', b'd', b'i', b'o', - b'-', b'b', b'o', b'u', b'n', b'd', b'a', b'r', b'y', b'X', 13, 10, 1, 2, 3, 128, - ]; - let mut body = format!( - "--{BOUNDARY}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"voice; name=model\"\r\nContent-Type: audio/wav\r\n\r\n" - ) - .into_bytes(); - body.extend_from_slice(&file_bytes); - body.extend_from_slice( - format!( - "\r\n--{BOUNDARY}\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\nauto\r\n--{BOUNDARY}--\r\n" - ) - .as_bytes(), - ); - let headers = format!( - "POST /v1/audio/transcriptions HTTP/1.1\r\nHost: localhost\r\nContent-Type: multipart/form-data; boundary=\"{BOUNDARY}\"\r\nContent-Length: {}\r\n\r\n", - body.len() - ) - .into_bytes(); - - let mut request = read_request_from_parts(vec![headers, body]).await; - assert_eq!(request.model_name.as_deref(), Some("auto")); - - rewrite_model_field(&mut request, "whisper-local"); - - assert_eq!(request.model_name.as_deref(), Some("whisper-local")); - assert!( - request - .raw - .windows(file_bytes.len()) - .any(|window| window == file_bytes) - ); - let header_end = request - .raw - .windows(4) - .position(|window| window == b"\r\n\r\n") - .unwrap() - + 4; - let content_type = format!("multipart/form-data; boundary={BOUNDARY}"); - assert_eq!( - multipart_model_field(&content_type, &request.raw[header_end..]) - .unwrap() - .as_deref(), - Some("whisper-local") - ); - let declared = std::str::from_utf8(&request.raw[..header_end]) - .unwrap() - .lines() - .find_map(|line| { - line.split_once(':') - .filter(|(name, _)| name.eq_ignore_ascii_case("content-length")) - .and_then(|(_, value)| value.trim().parse::().ok()) - }) - .unwrap(); - assert_eq!(declared, request.raw.len() - header_end); - assert_eq!(declared, request.body_len_bytes); -} - -#[test] -/// Reject ambiguous framing and unbounded model identifiers before route selection. -fn multipart_parser_rejects_invalid_boundaries_and_oversized_model_values() { - assert!(multipart_boundary("multipart/form-data; boundary=bad space").is_none()); - assert!(multipart_boundary("application/json; boundary=mesh").is_none()); - - let boundary = "mesh"; - let body = format!( - "--{boundary}\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\n{}\r\n--{boundary}--\r\n", - "x".repeat(257) - ); - assert!( - multipart_model_field( - &format!("multipart/form-data; boundary={boundary}"), - body.as_bytes() - ) - .unwrap() - .is_none() - ); -} - -#[tokio::test] -/// Two model fields cannot disagree about the destination of one upload. -async fn duplicate_multipart_model_is_rejected_before_audio_routing() { - let boundary = "mesh-audio-boundary"; - let body = format!( - "--{boundary}\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\nauto\r\n\ - --{boundary}\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\nother-model\r\n\ - --{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"voice.wav\"\r\n\r\nWAVE\r\n\ - --{boundary}--\r\n" - ); - let content_type = format!("multipart/form-data; boundary={boundary}"); - assert!( - multipart_model_field(&content_type, body.as_bytes()) - .unwrap_err() - .to_string() - .contains("duplicate multipart model field") - ); - - for path in ["/v1/audio/transcriptions", "/v1/audio/translations"] { - let request = format!( - "POST {path} HTTP/1.1\r\nHost: localhost\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\n\r\n{body}", - body.len() - ); - let (mut client, mut server) = tokio::io::duplex(request.len() + 1); - client.write_all(request.as_bytes()).await.unwrap(); - client.shutdown().await.unwrap(); - let error = read_http_request_with_plugin_manager_with_context(&mut server, None) - .await - .unwrap_err(); - assert_eq!(error.context().unwrap().client_path, path); - assert!( - error - .to_string() - .contains("duplicate multipart model field") - ); - } -} - -#[test] -/// A boundary-like sequence inside payload data is not a valid multipart start. -fn multipart_model_scanner_rejects_non_initial_boundary() { - let body = b"binary--mesh\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\nauto\r\n--mesh--\r\n"; - let error = multipart_model_field("multipart/form-data; boundary=mesh", body).unwrap_err(); - assert!( - error - .to_string() - .contains("must start with its declared boundary") - ); -} - -#[test] -/// Quoted filenames cannot impersonate the disposition's model-field parameter. -fn multipart_disposition_ignores_name_like_text_inside_quoted_filename() { - assert!( - !multipart_part_is_model( - "Content-Disposition: form-data; name=\"file\"; filename=\"voice; name=model\"" - ) - .unwrap() - ); - assert!(multipart_part_is_model("Content-Disposition: form-data; name=\"model\"").unwrap()); - assert!( - multipart_part_is_model("Content-Disposition: form-data; name=\"file\"; name=\"model\"") - .is_err() - ); -} - #[test] fn artifact_media_kind_is_closed_to_parsed_openai_json_routes() { let request = |path: &str, body: Option<&[u8]>| BufferedHttpRequest { diff --git a/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/routing.rs b/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/routing.rs index 0fe15ccf4d..3ad42c4b95 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/routing.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/routing.rs @@ -65,6 +65,7 @@ impl crate::network::metrics::RoutingTelemetrySink for PromptShapeSink { } } +/// Construct a reachable peer advertising the supplied model for routing tests. pub(super) fn test_peer_serving_model(peer_id: iroh::EndpointId, model: &str) -> mesh::PeerInfo { mesh::PeerInfo { id: peer_id, @@ -339,6 +340,7 @@ async fn remote_tokenizer_plan_routes_identity_model_without_context_rejection() } #[tokio::test] +/// Keep audio payload bytes out of text-context routing estimates. async fn remote_audio_upload_ignores_encoded_bytes_as_context_tokens() -> Result<()> { let model = "acme/audio-model:Q4_K_M"; let peer_id = iroh::EndpointId::from(iroh::SecretKey::generate().public()); diff --git a/crates/mesh-llm-host-runtime/src/protocol/convert.rs b/crates/mesh-llm-host-runtime/src/protocol/convert.rs index 10613780ba..9174c2a199 100644 --- a/crates/mesh-llm-host-runtime/src/protocol/convert.rs +++ b/crates/mesh-llm-host-runtime/src/protocol/convert.rs @@ -9,6 +9,8 @@ use anyhow::{Context, Result}; use iroh::{EndpointAddr, EndpointId}; use std::collections::{HashMap, HashSet}; +mod served_descriptors; + fn skippy_stage_subprotocols( artifact_transfer_supported: bool, stage_protocol_generation_supported: bool, @@ -295,18 +297,6 @@ fn proto_identity_to_local( } } -fn legacy_descriptor_from_identity( - identity: &crate::proto::node::ServedModelIdentity, -) -> crate::mesh::ServedModelDescriptor { - crate::mesh::ServedModelDescriptor { - identity: proto_identity_to_local(identity), - capabilities_known: false, - capabilities: crate::models::ModelCapabilities::default(), - topology: None, - metadata: None, - } -} - fn local_model_metadata_to_proto( metadata: &crate::mesh::ServedModelMetadata, ) -> crate::proto::node::ServedModelMetadata { @@ -351,6 +341,7 @@ fn proto_model_metadata_to_local( } } +/// Encode workload classes using additive protobuf discriminants. fn local_workload_class_to_proto(workload: crate::mesh::ModelWorkloadClass) -> i32 { use crate::mesh::ModelWorkloadClass as Local; use crate::proto::node::ModelWorkloadClass as Proto; @@ -367,6 +358,7 @@ fn local_workload_class_to_proto(workload: crate::mesh::ModelWorkloadClass) -> i } } +/// Distinguish legacy-unspecified workload metadata from unknown future values. fn proto_workload_class_to_local(value: i32) -> Option { use crate::mesh::ModelWorkloadClass as Local; use crate::proto::node::ModelWorkloadClass as Proto; @@ -559,19 +551,6 @@ fn proto_gpu_info_to_legacy_fields(gpus: &[crate::proto::node::GpuInfo]) -> Lega } } -/// Returns `true` when a proto descriptor carries a non-empty model name. -/// Descriptors without a valid identity are discarded. A non-empty descriptor -/// list is authoritative, so invalid entries never regain legacy semantics. -fn proto_descriptor_has_valid_identity( - descriptor: &crate::proto::node::ServedModelDescriptor, -) -> bool { - descriptor - .identity - .as_ref() - .map(|id| !id.model_name.is_empty()) - .unwrap_or(false) -} - pub(crate) fn sanitize_gossip_announcement_for_wire(ann: &PeerAnnouncement) -> PeerAnnouncement { let mut sanitized = ann.clone(); sanitized.available_models.clear(); @@ -1052,69 +1031,7 @@ pub(crate) fn proto_ann_to_local( .iter() .map(proto_runtime_descriptor_to_local) .collect(), - served_model_descriptors: if !pa.served_model_descriptors.is_empty() { - let descriptors: Vec<_> = - pa.served_model_descriptors - .iter() - .filter_map(|descriptor| { - if !proto_descriptor_has_valid_identity(descriptor) { - return None; - } - let metadata = descriptor - .metadata - .as_ref() - .map(proto_model_metadata_to_local); - let capabilities = descriptor - .capabilities - .as_ref() - .map(|caps| crate::models::ModelCapabilities { - multimodal: caps.multimodal, - vision: proto_capability_level_to_local(caps.vision), - audio: proto_capability_level_to_local(caps.audio), - reasoning: proto_capability_level_to_local(caps.reasoning), - tool_use: proto_capability_level_to_local(caps.tool_use), - moe: caps.moe, - }) - .unwrap_or_default(); - Some(crate::mesh::ServedModelDescriptor { - identity: descriptor - .identity - .as_ref() - .map(proto_identity_to_local) - .unwrap_or_default(), - capabilities_known: descriptor.capabilities_known.unwrap_or( - capabilities != crate::models::ModelCapabilities::default(), - ), - capabilities, - topology: descriptor.topology.as_ref().map(|topology| { - crate::models::ModelTopology { - moe: topology.moe.as_ref().map(|moe| { - crate::models::ModelMoeInfo { - expert_count: moe.expert_count, - used_expert_count: moe.used_expert_count, - min_experts_per_node: moe.min_experts_per_node, - source: moe.source.clone(), - ranking_source: moe.ranking_source.clone(), - ranking_origin: moe.ranking_origin.clone(), - ranking: moe.ranking.clone(), - ranking_prompt_count: moe.ranking_prompt_count, - ranking_tokens: moe.ranking_tokens, - ranking_layer_scope: moe.ranking_layer_scope.clone(), - } - }), - } - }), - metadata, - }) - }) - .collect(); - descriptors - } else { - pa.served_model_identities - .iter() - .map(legacy_descriptor_from_identity) - .collect() - }, + served_model_descriptors: Vec::new(), owner_attestation: pa .owner_attestation .as_ref() @@ -1155,7 +1072,7 @@ pub(crate) fn proto_ann_to_local( .as_ref() .and_then(proto_cache_affinity_to_local), }; - crate::mesh::backfill_legacy_descriptors(&mut ann); + served_descriptors::restore_served_descriptors(pa, &mut ann); ann.advertised_model_throughput = sanitize_model_throughput_hints_for_ann(&ann); ann.cache_affinity = sanitize_cache_affinity_for_ann(&ann); Some((addr, ann)) @@ -1395,6 +1312,7 @@ mod tests { use crate::mesh::requirements::peer_release_attestation_status; #[test] + /// Preserve every supported workload class across protobuf announcement conversion. fn workload_class_round_trips_through_additive_proto_metadata() { for workload in [ crate::mesh::ModelWorkloadClass::CausalGeneration, @@ -1419,6 +1337,7 @@ mod tests { } #[test] + /// Accept legacy unspecified classes while rejecting unknown modern classes. fn absent_proto_workload_class_is_legacy_compatible_but_unknown_is_rejected() { let absent = crate::proto::node::ServedModelMetadata::default(); assert_eq!(proto_model_metadata_to_local(&absent).workload_class, None); @@ -1448,6 +1367,7 @@ mod tests { } #[test] + /// Prevent unsupported workload descriptors from regaining routes through legacy identities. fn unknown_descriptor_workload_class_does_not_fall_back_to_legacy_identity() { let identity = crate::proto::node::ServedModelIdentity { model_name: "future-workload".to_string(), diff --git a/crates/mesh-llm-host-runtime/src/protocol/convert/served_descriptors.rs b/crates/mesh-llm-host-runtime/src/protocol/convert/served_descriptors.rs new file mode 100644 index 0000000000..17855dfad1 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/protocol/convert/served_descriptors.rs @@ -0,0 +1,191 @@ +//! Restore authoritative model descriptors without reviving rejected legacy routes. + +use super::{ + proto_capability_level_to_local, proto_identity_to_local, proto_model_metadata_to_local, +}; +use crate::mesh::{PeerAnnouncement, ServedModelDescriptor}; +use crate::proto::node; + +/// Only peers that omit descriptors may use identity/name-based legacy inference. +pub(super) fn restore_served_descriptors( + source: &node::PeerAnnouncement, + ann: &mut PeerAnnouncement, +) { + if source.served_model_descriptors.is_empty() { + ann.served_model_descriptors = source + .served_model_identities + .iter() + .map(legacy_descriptor_from_identity) + .collect(); + crate::mesh::backfill_legacy_descriptors(ann); + } else { + ann.served_model_descriptors = source + .served_model_descriptors + .iter() + .filter_map(proto_descriptor_to_local) + .collect(); + } +} + +/// Convert a legacy identity without inventing capability or workload metadata. +fn legacy_descriptor_from_identity( + identity: &crate::proto::node::ServedModelIdentity, +) -> crate::mesh::ServedModelDescriptor { + crate::mesh::ServedModelDescriptor { + identity: proto_identity_to_local(identity), + capabilities_known: false, + capabilities: crate::models::ModelCapabilities::default(), + topology: None, + metadata: None, + } +} + +/// Retain complete metadata for valid descriptors and discard missing/empty identities. +fn proto_descriptor_to_local( + descriptor: &node::ServedModelDescriptor, +) -> Option { + let identity = descriptor + .identity + .as_ref() + .filter(|id| !id.model_name.is_empty())?; + let capabilities = descriptor + .capabilities + .as_ref() + .map(|caps| crate::models::ModelCapabilities { + multimodal: caps.multimodal, + vision: proto_capability_level_to_local(caps.vision), + audio: proto_capability_level_to_local(caps.audio), + reasoning: proto_capability_level_to_local(caps.reasoning), + tool_use: proto_capability_level_to_local(caps.tool_use), + moe: caps.moe, + }) + .unwrap_or_default(); + Some(ServedModelDescriptor { + identity: proto_identity_to_local(identity), + capabilities_known: descriptor + .capabilities_known + .unwrap_or(capabilities != crate::models::ModelCapabilities::default()), + capabilities, + topology: descriptor + .topology + .as_ref() + .map(|topology| crate::models::ModelTopology { + moe: topology + .moe + .as_ref() + .map(|moe| crate::models::ModelMoeInfo { + expert_count: moe.expert_count, + used_expert_count: moe.used_expert_count, + min_experts_per_node: moe.min_experts_per_node, + source: moe.source.clone(), + ranking_source: moe.ranking_source.clone(), + ranking_origin: moe.ranking_origin.clone(), + ranking: moe.ranking.clone(), + ranking_prompt_count: moe.ranking_prompt_count, + ranking_tokens: moe.ranking_tokens, + ranking_layer_scope: moe.ranking_layer_scope.clone(), + }), + }), + metadata: descriptor + .metadata + .as_ref() + .map(proto_model_metadata_to_local), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::protocol::convert::proto_ann_to_local; + + /// Include both legacy fallback sources so malformed descriptors cannot hide the regression. + fn announcement_with_legacy_routes() -> node::PeerAnnouncement { + node::PeerAnnouncement { + endpoint_id: vec![1; 32], + serving_models: vec!["legacy-model".to_string()], + model_source: Some("legacy-model".to_string()), + served_model_identities: vec![node::ServedModelIdentity { + model_name: "legacy-model".to_string(), + ..Default::default() + }], + ..Default::default() + } + } + + /// A present-but-invalid descriptor list must never recover a legacy model route. + #[test] + fn invalid_descriptors_do_not_backfill_from_legacy_names_or_identities() { + for descriptor in [ + node::ServedModelDescriptor::default(), + node::ServedModelDescriptor { + identity: Some(node::ServedModelIdentity::default()), + ..Default::default() + }, + ] { + let source = node::PeerAnnouncement { + served_model_descriptors: vec![descriptor], + ..announcement_with_legacy_routes() + }; + let (_, ann) = proto_ann_to_local(&source).unwrap(); + assert!(ann.served_model_descriptors.is_empty()); + } + } + + /// Older identity-aware peers still restore the identities they actually advertise. + #[test] + fn absent_descriptors_preserve_legacy_identity_fallback() { + let (_, ann) = proto_ann_to_local(&announcement_with_legacy_routes()).unwrap(); + assert_eq!(ann.served_model_descriptors.len(), 1); + let descriptor = &ann.served_model_descriptors[0]; + assert_eq!(descriptor.identity.model_name, "legacy-model"); + assert!(!descriptor.capabilities_known); + assert!(descriptor.metadata.is_none()); + } + + /// Name-only peers retain the original primary-model inference contract. + #[test] + fn absent_descriptors_and_identities_preserve_legacy_name_fallback() { + let source = node::PeerAnnouncement { + served_model_identities: Vec::new(), + ..announcement_with_legacy_routes() + }; + let (_, ann) = proto_ann_to_local(&source).unwrap(); + assert_eq!(ann.served_model_descriptors.len(), 1); + let descriptor = &ann.served_model_descriptors[0]; + assert_eq!(descriptor.identity.model_name, "legacy-model"); + assert!(descriptor.identity.is_primary); + assert!(descriptor.metadata.is_none()); + } + + /// Partial invalidity must not drop valid workload metadata or restore extra legacy names. + #[test] + fn valid_descriptors_remain_authoritative_among_invalid_entries() { + let source = node::PeerAnnouncement { + served_model_descriptors: vec![ + node::ServedModelDescriptor::default(), + node::ServedModelDescriptor { + identity: Some(node::ServedModelIdentity { + model_name: "embedding-model".to_string(), + ..Default::default() + }), + capabilities_known: Some(true), + metadata: Some(node::ServedModelMetadata { + workload_class: Some(node::ModelWorkloadClass::Embedding as i32), + ..Default::default() + }), + ..Default::default() + }, + ], + ..announcement_with_legacy_routes() + }; + let (_, ann) = proto_ann_to_local(&source).unwrap(); + assert_eq!(ann.served_model_descriptors.len(), 1); + let descriptor = &ann.served_model_descriptors[0]; + assert_eq!(descriptor.identity.model_name, "embedding-model"); + assert!(descriptor.capabilities_known); + assert_eq!( + descriptor.metadata.as_ref().unwrap().workload_class, + Some(crate::mesh::ModelWorkloadClass::Embedding) + ); + } +} diff --git a/crates/openai-frontend/src/audio.rs b/crates/openai-frontend/src/audio.rs index 3358139eb7..4e492fce4f 100644 --- a/crates/openai-frontend/src/audio.rs +++ b/crates/openai-frontend/src/audio.rs @@ -138,6 +138,7 @@ pub struct AudioTranscriptionResponse { mod tests { use super::*; + /// Build a valid transcription fixture for multipart parameter tests. fn transcription(temperature: Option) -> AudioTranscriptionRequest { AudioTranscriptionRequest { model: "fixture".to_string(), @@ -151,6 +152,7 @@ mod tests { } #[test] + /// Reject non-finite and out-of-range transcription temperatures. fn transcription_temperature_is_bounded_to_openai_range() { for temperature in [None, Some(0.0), Some(0.5), Some(1.0)] { assert!(transcription(temperature).validate().is_ok()); diff --git a/crates/openai-frontend/src/embeddings.rs b/crates/openai-frontend/src/embeddings.rs index 96a900f2d9..f163817760 100644 --- a/crates/openai-frontend/src/embeddings.rs +++ b/crates/openai-frontend/src/embeddings.rs @@ -170,11 +170,13 @@ mod tests { use super::*; #[test] + /// Verify embeddings use little-endian float32 base64 encoding. fn base64_encoding_is_little_endian_f32() { assert_eq!(encode_f32_base64(&[1.0, -2.0]), "AACAPwAAAMA="); } #[test] + /// Reject embedding requests without inputs or with unsupported encodings. fn request_rejects_empty_batches_and_unknown_formats() { let mut request = EmbeddingsRequest { model: "embed".into(), @@ -190,6 +192,7 @@ mod tests { } #[test] + /// Reject negative embedding token IDs before backend execution. fn request_rejects_negative_token_ids() { let request = EmbeddingsRequest { model: "embed".into(), diff --git a/crates/openai-frontend/src/guardrails/compact.rs b/crates/openai-frontend/src/guardrails/compact.rs index 1f183b76df..fc28c94602 100644 --- a/crates/openai-frontend/src/guardrails/compact.rs +++ b/crates/openai-frontend/src/guardrails/compact.rs @@ -114,6 +114,7 @@ impl OpenAiBackend for CompactingOpenAiBackend { self.backend.completion_stream(request, context).await } + /// Forward embeddings and request context without chat processing. async fn embeddings( &self, request: EmbeddingsRequest, @@ -122,6 +123,7 @@ impl OpenAiBackend for CompactingOpenAiBackend { self.backend.embeddings(request, context).await } + /// Forward reranking and request context without chat processing. async fn rerank( &self, request: RerankRequest, @@ -130,6 +132,7 @@ impl OpenAiBackend for CompactingOpenAiBackend { self.backend.rerank(request, context).await } + /// Forward speech generation and request context unchanged. async fn audio_speech( &self, request: AudioSpeechRequest, @@ -138,6 +141,7 @@ impl OpenAiBackend for CompactingOpenAiBackend { self.backend.audio_speech(request, context).await } + /// Forward multipart transcription and request context unchanged. async fn audio_transcription( &self, request: AudioTranscriptionRequest, @@ -146,6 +150,7 @@ impl OpenAiBackend for CompactingOpenAiBackend { self.backend.audio_transcription(request, context).await } + /// Forward multipart translation and request context unchanged. async fn audio_translation( &self, request: AudioTranscriptionRequest, diff --git a/crates/openai-frontend/src/guardrails/mod.rs b/crates/openai-frontend/src/guardrails/mod.rs index 71fe4a0377..f4fea73c7d 100644 --- a/crates/openai-frontend/src/guardrails/mod.rs +++ b/crates/openai-frontend/src/guardrails/mod.rs @@ -356,6 +356,7 @@ impl OpenAiBackend for GuardedOpenAiBackend { self.backend.completion_stream(request, context).await } + /// Forward embeddings and request context without chat processing. async fn embeddings( &self, request: EmbeddingsRequest, @@ -364,6 +365,7 @@ impl OpenAiBackend for GuardedOpenAiBackend { self.backend.embeddings(request, context).await } + /// Forward reranking and request context without chat processing. async fn rerank( &self, request: RerankRequest, @@ -372,6 +374,7 @@ impl OpenAiBackend for GuardedOpenAiBackend { self.backend.rerank(request, context).await } + /// Forward speech generation and request context unchanged. async fn audio_speech( &self, request: AudioSpeechRequest, @@ -380,6 +383,7 @@ impl OpenAiBackend for GuardedOpenAiBackend { self.backend.audio_speech(request, context).await } + /// Forward multipart transcription and request context unchanged. async fn audio_transcription( &self, request: AudioTranscriptionRequest, @@ -388,6 +392,7 @@ impl OpenAiBackend for GuardedOpenAiBackend { self.backend.audio_transcription(request, context).await } + /// Forward multipart translation and request context unchanged. async fn audio_translation( &self, request: AudioTranscriptionRequest, diff --git a/crates/openai-frontend/src/hooks.rs b/crates/openai-frontend/src/hooks.rs index 6444a0d51b..5d227008c0 100644 --- a/crates/openai-frontend/src/hooks.rs +++ b/crates/openai-frontend/src/hooks.rs @@ -615,6 +615,7 @@ impl OpenAiBackend for HookedOpenAiBackend { self.backend.completion_stream(request, context).await } + /// Forward embeddings and request context without chat processing. async fn embeddings( &self, request: EmbeddingsRequest, @@ -623,6 +624,7 @@ impl OpenAiBackend for HookedOpenAiBackend { self.backend.embeddings(request, context).await } + /// Forward reranking and request context without chat processing. async fn rerank( &self, request: RerankRequest, @@ -631,6 +633,7 @@ impl OpenAiBackend for HookedOpenAiBackend { self.backend.rerank(request, context).await } + /// Forward speech generation and request context unchanged. async fn audio_speech( &self, request: AudioSpeechRequest, @@ -639,6 +642,7 @@ impl OpenAiBackend for HookedOpenAiBackend { self.backend.audio_speech(request, context).await } + /// Forward multipart transcription and request context unchanged. async fn audio_transcription( &self, request: AudioTranscriptionRequest, @@ -647,6 +651,7 @@ impl OpenAiBackend for HookedOpenAiBackend { self.backend.audio_transcription(request, context).await } + /// Forward multipart translation and request context unchanged. async fn audio_translation( &self, request: AudioTranscriptionRequest, diff --git a/crates/openai-frontend/src/rerank.rs b/crates/openai-frontend/src/rerank.rs index 0a77718725..65d0024414 100644 --- a/crates/openai-frontend/src/rerank.rs +++ b/crates/openai-frontend/src/rerank.rs @@ -87,6 +87,7 @@ mod tests { use super::*; #[test] + /// Require textual content in rerank document objects. fn document_objects_require_text() { let document = RerankDocument::Object(serde_json::json!({"title": "missing"})); assert!(document.text().is_err()); diff --git a/crates/openai-frontend/src/router_tests.rs b/crates/openai-frontend/src/router_tests.rs index 6218cffbca..faf04f5e95 100644 --- a/crates/openai-frontend/src/router_tests.rs +++ b/crates/openai-frontend/src/router_tests.rs @@ -566,6 +566,7 @@ impl OpenAiBackend for FakeBackend { ]))) } + /// Return indexed embedding fixtures with the requested encoding. async fn embeddings( &self, request: EmbeddingsRequest, @@ -585,6 +586,7 @@ impl OpenAiBackend for FakeBackend { )) } + /// Return ranked fixtures honoring `top_n` and document inclusion. async fn rerank( &self, request: RerankRequest, @@ -609,6 +611,7 @@ impl OpenAiBackend for FakeBackend { }) } + /// Return a binary speech fixture with the requested content type. async fn audio_speech( &self, request: AudioSpeechRequest, @@ -620,6 +623,7 @@ impl OpenAiBackend for FakeBackend { ) } + /// Return a transcription fixture containing the upload length. async fn audio_transcription( &self, request: AudioTranscriptionRequest, @@ -630,6 +634,7 @@ impl OpenAiBackend for FakeBackend { }) } + /// Return a translation fixture containing the upload length. async fn audio_translation( &self, request: AudioTranscriptionRequest, diff --git a/crates/skippy-ffi/build.rs b/crates/skippy-ffi/build.rs index 1861cf2859..3e78fb494b 100644 --- a/crates/skippy-ffi/build.rs +++ b/crates/skippy-ffi/build.rs @@ -445,6 +445,7 @@ fn static_archive_exists( build_dir.join(unix_archive).exists() || build_dir.join(msvc_archive).exists() } +/// Read CMake booleans while tolerating platform line endings and whitespace. fn cmake_bool_enabled(cache: &std::path::Path, key: &str) -> bool { let Ok(contents) = std::fs::read_to_string(cache) else { return false; @@ -452,6 +453,7 @@ fn cmake_bool_enabled(cache: &std::path::Path, key: &str) -> bool { cmake_cache_bool(&contents, key) } +/// Resolve an enabled backend archive without trusting stale native build outputs. fn configured_backend_archive( build_dir: &std::path::Path, cmake_cache: &std::path::Path, diff --git a/crates/skippy-ffi/src/abi.rs b/crates/skippy-ffi/src/abi.rs index e46c905d7d..ed04ed2928 100644 --- a/crates/skippy-ffi/src/abi.rs +++ b/crates/skippy-ffi/src/abi.rs @@ -62,6 +62,7 @@ pub struct WorkloadInfoV1 { } impl Default for WorkloadInfoV1 { + /// Initialize the workload descriptor with its ABI size and version. fn default() -> Self { Self { abi_version: WORKLOAD_INFO_V1_ABI_VERSION, diff --git a/crates/skippy-runtime/src/media.rs b/crates/skippy-runtime/src/media.rs index 12fad95ffd..7d785452c9 100644 --- a/crates/skippy-runtime/src/media.rs +++ b/crates/skippy-runtime/src/media.rs @@ -151,6 +151,7 @@ impl StageModel { self.media.is_some() } + /// Report whether the configured projector supports native audio generation. pub fn supports_speech_synthesis(&self) -> bool { self.media.as_ref().is_some_and(|projector| { let info = unsafe { skippy_ffi::mtmd_gen_audio_get_info(projector.raw) }; @@ -873,6 +874,7 @@ impl StageModel { } } +/// Validate and quantize native float32 PCM into clamped signed little-endian samples. fn pcm_f32_to_s16le(bytes: &[u8]) -> Result> { if !bytes.len().is_multiple_of(std::mem::size_of::()) { return Err(anyhow!("native PCM payload is not aligned to f32 samples")); @@ -893,6 +895,7 @@ mod tests { use super::pcm_f32_to_s16le; #[test] + /// Verify clipping and quantization at signed PCM boundaries. fn pcm_conversion_clamps_and_quantizes_native_float_samples() { let samples = [-2.0_f32, -1.0, -0.5, 0.0, 0.5, 1.0, 2.0]; let bytes = samples @@ -915,6 +918,7 @@ mod tests { } #[test] + /// Reject PCM payloads without complete float32 samples. fn pcm_conversion_rejects_misaligned_native_payload() { assert!(pcm_f32_to_s16le(&[0, 1, 2]).is_err()); } diff --git a/crates/skippy-runtime/src/media/speech_session_tests.rs b/crates/skippy-runtime/src/media/speech_session_tests.rs index e38c985cd1..74e2c31b71 100644 --- a/crates/skippy-runtime/src/media/speech_session_tests.rs +++ b/crates/skippy-runtime/src/media/speech_session_tests.rs @@ -3,6 +3,7 @@ use super::*; use crate::{ModelInfo, RuntimeConfig, TensorRole}; +/// Load the opt-in speech model and projector without a synthetic runtime fallback. fn speech_fixture() -> Result> { if std::env::var("SKIPPY_WORKLOAD_CLASS").as_deref() != Ok("speech_synthesis") { return Ok(None); @@ -32,6 +33,7 @@ fn speech_fixture() -> Result> { StageModel::open(path, &config).map(Some) } +/// Prove a speech session can resume ordinary decoding with a real output row. fn assert_generation_reusable(model: &StageModel, session: &mut StageSession) -> Result<()> { session.reset()?; let tokens = model.tokenize("The mesh is ready.", true)?; @@ -46,6 +48,7 @@ fn assert_generation_reusable(model: &StageModel, session: &mut StageSession) -> } #[test] +/// Verify all speech exit paths restore ordinary session decoding. fn speech_success_cancellation_and_native_failure_leave_session_reusable() -> Result<()> { let Some(model) = speech_fixture()? else { return Ok(()); diff --git a/crates/skippy-runtime/src/native.rs b/crates/skippy-runtime/src/native.rs index cbd45de005..e6fd8cdcb9 100644 --- a/crates/skippy-runtime/src/native.rs +++ b/crates/skippy-runtime/src/native.rs @@ -50,6 +50,7 @@ pub struct WorkloadInfo { impl TryFrom for WorkloadInfo { type Error = anyhow::Error; + /// Validate the native descriptor layout and translate supported workload and pooling values. fn try_from(raw: skippy_ffi::WorkloadInfoV1) -> Result { if raw.abi_version != skippy_ffi::WORKLOAD_INFO_V1_ABI_VERSION || raw.struct_size != std::mem::size_of::() as u32 @@ -1075,6 +1076,7 @@ mod output_capacity_tests { }; #[test] + /// Cover all supported native workload and pooling discriminants. fn workload_descriptor_converts_all_native_classes_and_pooling_modes() { let cases = [ ( @@ -1115,6 +1117,7 @@ mod output_capacity_tests { } #[test] + /// Reject incompatible native descriptor versions and sizes. fn workload_descriptor_rejects_incompatible_layout_versions() { let invalid_version = skippy_ffi::WorkloadInfoV1 { abi_version: skippy_ffi::WORKLOAD_INFO_V1_ABI_VERSION + 1, diff --git a/crates/skippy-server/src/frontend/backend.rs b/crates/skippy-server/src/frontend/backend.rs index 85938d67cf..3e3fddbb31 100644 --- a/crates/skippy-server/src/frontend/backend.rs +++ b/crates/skippy-server/src/frontend/backend.rs @@ -1131,6 +1131,7 @@ impl OpenAiBackend for StageOpenAiBackend { }))) } + /// Execute embeddings only through an admitted local full-model workload. async fn embeddings( &self, request: EmbeddingsRequest, @@ -1186,6 +1187,7 @@ impl OpenAiBackend for StageOpenAiBackend { )) } + /// Execute reranking only through an admitted local full-model workload. async fn rerank( &self, request: RerankRequest, @@ -1241,6 +1243,7 @@ impl OpenAiBackend for StageOpenAiBackend { }) } + /// Execute native speech synthesis with the configured local projector. async fn audio_speech( &self, request: AudioSpeechRequest, @@ -1307,6 +1310,7 @@ impl OpenAiBackend for StageOpenAiBackend { AudioResponse::new(audio.bytes, content_type) } + /// Decode uploaded audio using the local speech-recognition runtime. async fn audio_transcription( &self, request: AudioTranscriptionRequest, @@ -1315,6 +1319,7 @@ impl OpenAiBackend for StageOpenAiBackend { self.audio_to_text(request, false, context).await } + /// Translate uploaded audio using the local speech-recognition runtime. async fn audio_translation( &self, request: AudioTranscriptionRequest, diff --git a/crates/skippy-server/src/frontend/backend/tests.rs b/crates/skippy-server/src/frontend/backend/tests.rs index afdf9a7213..5d75310ff4 100644 --- a/crates/skippy-server/src/frontend/backend/tests.rs +++ b/crates/skippy-server/src/frontend/backend/tests.rs @@ -1249,6 +1249,7 @@ fn hooks_test_backend(hook_policy: Option>) -> StageOp } } +/// Construct a stage-zero embedded topology for non-chat admission tests. fn embedded_non_chat_test_mode(config: skippy_protocol::StageConfig) -> OpenAiBackendMode { OpenAiBackendMode::EmbeddedStageZero { config, @@ -1263,6 +1264,7 @@ fn embedded_non_chat_test_mode(config: skippy_protocol::StageConfig) -> OpenAiBa } #[test] +/// Allow non-chat execution for a complete, unsplit embedded model. fn embedded_stage_zero_admits_unsplit_local_non_chat_topology() { let mut backend = hooks_test_backend(None); backend.mode = embedded_non_chat_test_mode(backend.config.clone()); @@ -1271,6 +1273,7 @@ fn embedded_stage_zero_admits_unsplit_local_non_chat_topology() { } #[test] +/// Reject non-chat execution with incomplete layer, tensor, or expert ownership. fn non_chat_topology_guard_rejects_staged_and_filtered_models() { let mut backend = hooks_test_backend(None); let full_config = backend.config.clone(); diff --git a/crates/skippy-server/src/frontend/generation_flow/encoder_decoder.rs b/crates/skippy-server/src/frontend/generation_flow/encoder_decoder.rs index cac379dcf1..9946e80cbc 100644 --- a/crates/skippy-server/src/frontend/generation_flow/encoder_decoder.rs +++ b/crates/skippy-server/src/frontend/generation_flow/encoder_decoder.rs @@ -13,6 +13,7 @@ use super::LocalSessionCleanupGuard; impl StageOpenAiBackend { #[allow(clippy::too_many_arguments)] + /// Encode once, decode bounded tokens, and clean up the exclusive session on every exit. pub(super) fn generate_encoder_decoder_tokens( &self, prompt_token_ids: &[i32], diff --git a/crates/skippy-server/src/frontend/tests/non_chat.rs b/crates/skippy-server/src/frontend/tests/non_chat.rs index 8127c9d97f..b5126e000d 100644 --- a/crates/skippy-server/src/frontend/tests/non_chat.rs +++ b/crates/skippy-server/src/frontend/tests/non_chat.rs @@ -2,7 +2,7 @@ use super::*; use openai_frontend::{ AudioFormat, AudioSpeechRequest, AudioTranscriptionRequest, EmbeddingInput, EmbeddingOutput, - EmbeddingsRequest, RerankDocument, RerankRequest, + EmbeddingsRequest, RerankDocument, RerankRequest, RerankResult, }; use skippy_runtime::ModelWorkload; @@ -170,6 +170,7 @@ fn workload_stage_config(fixture: &WorkloadFixture) -> StageConfig { /// Compare vector dimensions and each component with a bounded numeric tolerance. fn assert_vectors_close(left: &[f32], right: &[f32]) { assert_eq!(left.len(), right.len()); + assert!(left.iter().chain(right).all(|value| value.is_finite())); let maximum_delta = left .iter() .zip(right) @@ -178,6 +179,74 @@ fn assert_vectors_close(left: &[f32], right: &[f32]) { assert!(maximum_delta <= 1e-5, "embedding delta {maximum_delta}"); } +/// Distinct inputs must not collapse to one otherwise well-formed embedding vector. +fn require_distinct_embeddings(left: &EmbeddingOutput, right: &EmbeddingOutput) -> Result<()> { + let (EmbeddingOutput::Float(left), EmbeddingOutput::Float(right)) = (left, right) else { + bail!("float embedding request returned a non-float payload"); + }; + anyhow::ensure!(left.len() == right.len(), "embedding dimensions differ"); + anyhow::ensure!( + left.iter().chain(right).all(|value| value.is_finite()), + "embedding contains non-finite values" + ); + anyhow::ensure!( + left.iter() + .zip(right) + .any(|(left, right)| (left - right).abs() > 1e-5), + "distinct inputs returned identical embeddings" + ); + Ok(()) +} + +/// The relevant fixture must strictly outrank the unrelated fixture in response order. +fn require_relevant_document_first(results: &[RerankResult]) -> Result<()> { + anyhow::ensure!( + results.len() == 2 && results[0].index == 0 && results[1].index == 1, + "rerank did not place the relevant document first" + ); + anyhow::ensure!( + results + .iter() + .all(|result| result.relevance_score.is_finite()) + && results[0].relevance_score > results[1].relevance_score, + "the relevant document did not outrank the unrelated document" + ); + Ok(()) +} + +/// A constant normalized vector is not evidence that embeddings depend on their inputs. +#[test] +fn embedding_certification_rejects_constant_vectors() { + let query = EmbeddingOutput::Float(vec![1.0, 0.0]); + assert!(require_distinct_embeddings(&query, &query).is_err()); + assert!(require_distinct_embeddings(&query, &EmbeddingOutput::Float(vec![0.0, 1.0])).is_ok()); +} + +/// Ties, reversed scores, and reordered indexes must not certify a broken ranker. +#[test] +fn rerank_certification_rejects_degenerate_or_misordered_results() { + let mut results = vec![ + RerankResult { + index: 0, + relevance_score: 0.9, + document: None, + }, + RerankResult { + index: 1, + relevance_score: 0.1, + document: None, + }, + ]; + assert!(require_relevant_document_first(&results).is_ok()); + for score in [0.1, 0.0, f32::NAN, f32::INFINITY] { + results[0].relevance_score = score; + assert!(require_relevant_document_first(&results).is_err()); + } + results[0].relevance_score = 0.9; + results.swap(0, 1); + assert!(require_relevant_document_first(&results).is_err()); +} + /// Check native vector shape, determinism and semantic separation for pinned inputs. async fn certify_embedding(backend: &StageOpenAiBackend) -> Result<()> { let info = backend.ensure_local_workload(ModelWorkload::Embedding)?; @@ -213,6 +282,8 @@ async fn certify_embedding(backend: &StageOpenAiBackend) -> Result<()> { assert!((norm - 1.0).abs() <= 1e-4, "embedding norm {norm}"); assert_vectors_close(left, right); } + require_distinct_embeddings(&first.data[0].embedding, &first.data[1].embedding)?; + require_distinct_embeddings(&second.data[0].embedding, &second.data[1].embedding)?; Ok(()) } @@ -243,6 +314,8 @@ async fn certify_rerank(backend: &StageOpenAiBackend) -> Result<()> { assert!((left.relevance_score - right.relevance_score).abs() <= 1e-6); assert!(left.document.is_some()); } + require_relevant_document_first(&first.results)?; + require_relevant_document_first(&second.results)?; Ok(()) } diff --git a/crates/skippy-server/src/frontend/tests/support.rs b/crates/skippy-server/src/frontend/tests/support.rs index 4819b7e662..a412cea330 100644 --- a/crates/skippy-server/src/frontend/tests/support.rs +++ b/crates/skippy-server/src/frontend/tests/support.rs @@ -117,6 +117,7 @@ pub(super) fn unsupported_code(error: OpenAiError) -> Option { error.body().error.code } +/// Create a real local OpenAI backend for the supplied native model and configuration. pub(super) fn local_openai_backend( config: StageConfig, model_id: impl Into, diff --git a/crates/skippy-server/src/http.rs b/crates/skippy-server/src/http.rs index e042aed461..2cf91f9dbd 100644 --- a/crates/skippy-server/src/http.rs +++ b/crates/skippy-server/src/http.rs @@ -818,6 +818,7 @@ mod tests { #[cfg(unix)] #[tokio::test] + /// Verify a completed server can immediately reuse its listening address. async fn serving_listener_can_rebind_after_server_closes_connection() -> Result<()> { use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -841,6 +842,7 @@ mod tests { } #[tokio::test] + /// Preserve exclusive address ownership while the first listener remains live. async fn serving_listener_rejects_another_live_listener() -> Result<()> { let listener = bind_serve_listener("127.0.0.1:0".parse()?)?; let error = bind_serve_listener(listener.local_addr()?) diff --git a/scripts/ci-openai-embeddings-smoke.py b/scripts/ci-openai-embeddings-smoke.py index ebf802b25a..1badc0a910 100755 --- a/scripts/ci-openai-embeddings-smoke.py +++ b/scripts/ci-openai-embeddings-smoke.py @@ -73,6 +73,9 @@ def main() -> None: values = struct.unpack(f"<{dimensions}f", raw) if not all(math.isfinite(value) for value in values): raise RuntimeError("base64 embedding contains a non-finite value") + if not all(math.isclose(value, expected, rel_tol=1e-5, abs_tol=1e-6) + for value, expected in zip(values, response.data[0].embedding)): + raise RuntimeError("base64 embedding differs from float response") print( f"openai-python embeddings smoke passed: model={args.model} " diff --git a/scripts/ci-openai-workload-smoke.py b/scripts/ci-openai-workload-smoke.py index e7c623b280..e62f9097b8 100644 --- a/scripts/ci-openai-workload-smoke.py +++ b/scripts/ci-openai-workload-smoke.py @@ -27,6 +27,7 @@ def request_json(base_url: str, path: str, payload: dict[str, object]) -> dict: + """POST JSON with a bounded timeout and require an object response.""" request = urllib.request.Request( f"{base_url}{path}", data=json.dumps(payload).encode("utf-8"), @@ -43,6 +44,7 @@ def request_json(base_url: str, path: str, payload: dict[str, object]) -> dict: def request_bytes(base_url: str, path: str, payload: dict[str, object]) -> tuple[str, bytes]: + """POST JSON with a bounded timeout and return the response media type and bytes.""" request = urllib.request.Request( f"{base_url}{path}", data=json.dumps(payload).encode("utf-8"), @@ -54,6 +56,7 @@ def request_bytes(base_url: str, path: str, payload: dict[str, object]) -> tuple def request_multipart(base_url: str, path: str, model: str, media_path: Path) -> dict: + """Upload a WAV fixture with the selected model and require a JSON object response.""" boundary = "mesh-llm-workload-smoke" body = ( f"--{boundary}\r\n" @@ -80,6 +83,7 @@ def request_multipart(base_url: str, path: str, model: str, media_path: Path) -> def smoke_embedding(base_url: str, model: str) -> None: + """Check vector metadata, normalization, semantic ordering, usage, and float/base64 parity.""" inputs = list(EMBEDDING_INPUTS) result = request_json( base_url, @@ -134,11 +138,16 @@ def smoke_embedding(base_url: str, model: str) -> None: raw = base64.b64decode(payload, validate=True) if len(raw) != len(vectors[0]) * struct.calcsize(" None: + """Require finite scores with the relevant document strictly ahead of its distractor.""" result = request_json( base_url, "/rerank", @@ -168,6 +177,7 @@ def smoke_rerank(base_url: str, model: str) -> None: def smoke_encoder_decoder(base_url: str, model: str) -> None: + """Verify a deterministic translation contains the expected language anchor.""" result = request_json( base_url, "/completions", @@ -190,6 +200,7 @@ def smoke_encoder_decoder(base_url: str, model: str) -> None: def smoke_ocr(base_url: str, model: str, media_path: Path) -> None: + """Submit the image fixture through chat and require a nonempty transcription.""" image = base64.b64encode(media_path.read_bytes()).decode("ascii") result = request_json( base_url, @@ -218,6 +229,7 @@ def smoke_ocr(base_url: str, model: str, media_path: Path) -> None: def smoke_speech_synthesis(base_url: str, model: str) -> None: + """Validate generated audio framing and nonempty sample content.""" content_type, audio = request_bytes( base_url, "/audio/speech", @@ -243,6 +255,7 @@ def smoke_speech_synthesis(base_url: str, model: str) -> None: def smoke_speech_recognition(base_url: str, model: str, media_path: Path) -> None: + """Submit the audio fixture and require a nonempty transcription.""" result = request_multipart(base_url, "/audio/transcriptions", model, media_path) text = result.get("text") if not isinstance(text, str) or not text.strip(): @@ -250,6 +263,7 @@ def smoke_speech_recognition(base_url: str, model: str, media_path: Path) -> Non def main() -> None: + """Dispatch the workload smoke with its required model and media fixtures.""" parser = argparse.ArgumentParser() parser.add_argument("--base-url", required=True) parser.add_argument("--model", required=True) diff --git a/scripts/ci-workload-monolithic-oracle.py b/scripts/ci-workload-monolithic-oracle.py index 2ab1785b87..0a6b9c447e 100644 --- a/scripts/ci-workload-monolithic-oracle.py +++ b/scripts/ci-workload-monolithic-oracle.py @@ -29,6 +29,7 @@ def request_json(base_url: str, path: str, payload: dict[str, object]) -> dict: + """POST JSON with a bounded timeout and require an object response.""" request = urllib.request.Request( f"{base_url}{path}", data=json.dumps(payload).encode("utf-8"), @@ -182,6 +183,7 @@ def compare_encoder_decoder(candidate: dict, reference: dict) -> str: def monolithic_completion(oracle_cli: str, model_path: str) -> dict: + """Run the independent CPU oracle and strip only its terminal end-of-text marker.""" command = [ oracle_cli, "-m", model_path, "-p", ENCODER_DECODER_PROMPT, "-n", "32", "-c", "0", "-b", "2048", "-ub", "2048", "-ngl", "0", @@ -206,6 +208,7 @@ def monolithic_completion(oracle_cli: str, model_path: str) -> dict: def main() -> None: + """Select the independent CLI or server oracle for the requested workload.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--candidate-url", required=True) parser.add_argument("--oracle-url") diff --git a/scripts/generate-ocr-oracle-fixture.py b/scripts/generate-ocr-oracle-fixture.py index e4e27ba5cb..fb0dc96936 100644 --- a/scripts/generate-ocr-oracle-fixture.py +++ b/scripts/generate-ocr-oracle-fixture.py @@ -31,11 +31,13 @@ def chunk(kind: bytes, data: bytes) -> bytes: + """Frame a PNG chunk with a network-order length and CRC.""" payload = kind + data return struct.pack(">I", len(data)) + payload + struct.pack(">I", zlib.crc32(payload)) def png_bytes() -> bytes: + """Render the original fixed-glyph OCR fixture as deterministic RGB PNG bytes.""" pixels = bytearray(b"\xff" * (WIDTH * HEIGHT * 3)) for char_index, char in enumerate(TEXT): for glyph_y, row in enumerate(GLYPHS[char]): @@ -61,6 +63,7 @@ def png_bytes() -> bytes: def main() -> None: + """Write the deterministic original OCR fixture to the requested path.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--output", required=True, type=Path) args = parser.parse_args() diff --git a/scripts/llama-oracle-source.py b/scripts/llama-oracle-source.py index 198af68e85..137de3a762 100644 --- a/scripts/llama-oracle-source.py +++ b/scripts/llama-oracle-source.py @@ -14,6 +14,7 @@ def ordered_patches(patch_dir: Path) -> list[Path]: + """Validate and order both patch series exactly as native source preparation does.""" patches = sorted(patch_dir.glob("*.patch")) for expected, patch in enumerate(patches, start=1): if not re.fullmatch(rf"{expected:04d}-.+\.patch", patch.name): @@ -38,6 +39,7 @@ def ordered_patches(patch_dir: Path) -> list[Path]: def patch_digest(patch_dir: Path) -> str: + """Hash patch names and contents in validated application order.""" digest = hashlib.sha256() for patch in ordered_patches(patch_dir): relative_name = patch.relative_to(patch_dir).as_posix() @@ -47,6 +49,7 @@ def patch_digest(patch_dir: Path) -> str: def prepared_patched_sha(root: Path) -> str: + """Verify the prepared checkout's upstream pin, patch digest, schema, and clean HEAD.""" checkout = root / ".deps/llama.cpp" prepared_upstream = (checkout / ".mesh-llm-upstream-sha").read_text(encoding="utf-8").strip() prepared_patch_digest = (checkout / ".mesh-llm-patch-digest").read_text(encoding="utf-8").strip() @@ -69,6 +72,7 @@ def prepared_patched_sha(root: Path) -> str: def main() -> None: + """Print the verified patched revision or explain its provenance mismatch.""" try: print(prepared_patched_sha(ROOT)) except (OSError, RuntimeError, subprocess.CalledProcessError) as error: diff --git a/scripts/skippy-ocr-asr-oracle.py b/scripts/skippy-ocr-asr-oracle.py index aa98d6770e..b9e8f81729 100644 --- a/scripts/skippy-ocr-asr-oracle.py +++ b/scripts/skippy-ocr-asr-oracle.py @@ -79,6 +79,7 @@ def compare_text(candidate: object, reference: object, expected: str | None, def request_json(base_url: str, path: str, payload: dict[str, object]) -> dict: + """POST JSON with a bounded timeout and require an object response.""" request = urllib.request.Request( f"{base_url.rstrip('/')}{path}", data=json.dumps(payload).encode("utf-8"), @@ -89,6 +90,7 @@ def request_json(base_url: str, path: str, payload: dict[str, object]) -> dict: def request_multipart(base_url: str, path: str, model: str, media: bytes) -> dict: + """Upload WAV with deterministic recognition parameters and a collision-checked boundary.""" if BOUNDARY.encode("ascii") in media: raise RuntimeError("audio fixture collides with multipart boundary") body = ( @@ -115,6 +117,7 @@ def request_multipart(base_url: str, path: str, model: str, media: bytes) -> dic def response_json(request: urllib.request.Request) -> dict: + """Read a bounded oracle response and require the JSON object media contract.""" with urllib.request.urlopen(request, timeout=240) as response: if response.headers.get_content_type() != "application/json": raise RuntimeError(f"{request.full_url} returned non-JSON content") @@ -125,6 +128,7 @@ def response_json(request: urllib.request.Request) -> dict: def chat_text(response: dict, source: str) -> object: + """Extract text from exactly one well-formed chat completion choice.""" choices = response.get("choices") if not isinstance(choices, list) or len(choices) != 1: raise RuntimeError(f"{source} returned invalid OCR choices") @@ -139,6 +143,7 @@ def chat_text(response: dict, source: str) -> object: def compare_ocr(candidate_url: str, oracle_url: str, model: str, image: bytes, expected: str) -> str: + """Compare candidate and oracle OCR with the independent fixture transcription.""" payload = { "model": model, "messages": [{ @@ -165,6 +170,7 @@ def compare_ocr(candidate_url: str, oracle_url: str, model: str, image: bytes, def compare_asr(candidate_url: str, oracle_url: str, model: str, audio: bytes, expected: str | None) -> str: + """Compare multipart ASR with the oracle's aligned chat-based audio prompt.""" # llama-server's /audio/transcriptions substitutes its own default user # instruction. Compare the actual Skippy audio route with monolithic chat # using the exact instruction and media ordering that Skippy constructs. @@ -194,6 +200,7 @@ def compare_asr(candidate_url: str, oracle_url: str, model: str, audio: bytes, def main() -> None: + """Validate media prerequisites and report class-specific oracle comparison evidence.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--candidate-url", required=True) parser.add_argument("--oracle-url", required=True) diff --git a/scripts/tests/test_check_skippy_workload_candidate.py b/scripts/tests/test_check_skippy_workload_candidate.py index 2df003351f..e4264c0393 100644 --- a/scripts/tests/test_check_skippy_workload_candidate.py +++ b/scripts/tests/test_check_skippy_workload_candidate.py @@ -21,6 +21,7 @@ class CandidateBuildFreshnessTests(unittest.TestCase): def test_producer_binds_source_native_stamp_and_every_executable(self) -> None: + """Reject producer evidence after source, native build, or executable replacement.""" with tempfile.TemporaryDirectory() as temp_dir: root = Path(temp_dir) native = root / "native" @@ -55,6 +56,7 @@ def test_producer_binds_source_native_stamp_and_every_executable(self) -> None: CANDIDATE.write_producer(manifest, binary, native, test_binary, snapshot) def _check(self, binary: Path, build_dir: Path) -> subprocess.CompletedProcess[str]: + """Run the production candidate checker against the isolated build fixture.""" return subprocess.run( [ sys.executable, @@ -70,6 +72,7 @@ def _check(self, binary: Path, build_dir: Path) -> subprocess.CompletedProcess[s ) def test_accepts_binary_linked_after_stamped_native_build(self) -> None: + """Accept a candidate linked after its recorded native build.""" with tempfile.TemporaryDirectory() as temp_dir: directory = Path(temp_dir) stamp = directory / ".mesh-llm-build-stamp" @@ -81,6 +84,7 @@ def test_accepts_binary_linked_after_stamped_native_build(self) -> None: self.assertEqual(0, self._check(binary, directory).returncode) def test_rejects_binary_older_than_or_equal_to_native_stamp(self) -> None: + """Reject stale candidates even when their executable paths still exist.""" with tempfile.TemporaryDirectory() as temp_dir: directory = Path(temp_dir) stamp = directory / ".mesh-llm-build-stamp" @@ -96,6 +100,7 @@ def test_rejects_binary_older_than_or_equal_to_native_stamp(self) -> None: self.assertIn("candidate executable predates", result.stderr) def test_rejects_missing_executable_or_stamp(self) -> None: + """Require the candidate executable and its native-build stamp.""" with tempfile.TemporaryDirectory() as temp_dir: directory = Path(temp_dir) binary = directory / "skippy-server" diff --git a/scripts/tests/test_ci_openai_embeddings_smoke.py b/scripts/tests/test_ci_openai_embeddings_smoke.py index 7cf06e705c..1b97886964 100644 --- a/scripts/tests/test_ci_openai_embeddings_smoke.py +++ b/scripts/tests/test_ci_openai_embeddings_smoke.py @@ -102,6 +102,21 @@ def test_invalid_payloads_still_fail(self) -> None: with self.assertRaisesRegex(RuntimeError, message): self.run_smoke(response) + def test_base64_values_must_match_float_response(self) -> None: + """A finite normalized vector of the right size still needs value parity.""" + response = self.encoded_response() + response.data[0].embedding = base64.b64encode(struct.pack("<2f", 0, 1)).decode() + with self.assertRaisesRegex(RuntimeError, "differs from float"): + self.run_smoke(response) + + def test_base64_parity_allows_float32_rounding(self) -> None: + """Float32 rounding must not reject otherwise equivalent representations.""" + response = self.encoded_response() + response.data[0].embedding = base64.b64encode( + struct.pack("<2f", 1 + 1e-7, 1e-7) + ).decode() + self.run_smoke(response) + class EmbeddingHttpSmokeTests(unittest.TestCase): """The raw HTTP smoke must enforce the same base64 envelope as the SDK smoke.""" @@ -131,6 +146,22 @@ def test_base64_cardinality_and_metadata_are_required(self) -> None: with self.subTest(changed=changed), self.assertRaises(RuntimeError): self.run_smoke({**good, **changed}) + def test_base64_values_must_match_float_response(self) -> None: + """Do not award HTTP certification for a different, correctly sized vector.""" + encoded = {"object": "list", "model": "fixture", "data": [{ + "object": "embedding", "index": 0, + "embedding": base64.b64encode(struct.pack("<2f", 0, 1)).decode(), + }]} + with self.assertRaisesRegex(RuntimeError, "differs from float"): + self.run_smoke(encoded) + + def test_base64_parity_allows_float32_rounding(self) -> None: + """Use the same rounding allowance as the official SDK validator.""" + self.run_smoke({"object": "list", "model": "fixture", "data": [{ + "object": "embedding", "index": 0, + "embedding": base64.b64encode(struct.pack("<2f", 1 + 1e-7, 1e-7)).decode(), + }]}) + if __name__ == "__main__": unittest.main() diff --git a/scripts/tests/test_llama_oracle_source.py b/scripts/tests/test_llama_oracle_source.py index 5dfb4970d8..69da36e22f 100644 --- a/scripts/tests/test_llama_oracle_source.py +++ b/scripts/tests/test_llama_oracle_source.py @@ -17,6 +17,7 @@ class LlamaOracleSourceTests(unittest.TestCase): def test_digest_includes_generated_series_in_prepare_order(self) -> None: + """Include generated patches in the digest using native preparation order.""" with tempfile.TemporaryDirectory() as temp_dir: patches = Path(temp_dir) (patches / "0001-base.patch").write_bytes(b"base\n") @@ -38,6 +39,7 @@ def test_digest_includes_generated_series_in_prepare_order(self) -> None: source.patch_digest(patches) def test_prepared_checkout_rejects_patch_queue_drift(self) -> None: + """Reject an oracle checkout after its source patch queue changes.""" with tempfile.TemporaryDirectory() as temp_dir: root = Path(temp_dir) checkout = root / ".deps/llama.cpp" diff --git a/scripts/tests/test_llama_upstream_canary_contract.py b/scripts/tests/test_llama_upstream_canary_contract.py index 4db1cc2ca1..9f46b80a5f 100644 --- a/scripts/tests/test_llama_upstream_canary_contract.py +++ b/scripts/tests/test_llama_upstream_canary_contract.py @@ -648,6 +648,7 @@ def test_battery_builds_once_then_skips_build_in_each_lane(self) -> None: ) def test_workload_dry_run_needs_no_oracle_and_forwards_startup_deadline(self) -> None: + """Keep planning independent of oracle availability while forwarding the startup deadline.""" model = self._model() model.update({ "class": "embedding", @@ -684,6 +685,7 @@ def test_dry_run_reconciles_every_planned_family(self) -> None: self.assertIn("--family second-family", commands[3]) def test_supplied_plan_cannot_omit_a_manifest_selected_family(self) -> None: + """Reject a supplied plan that drops a manifest-selected family.""" with tempfile.TemporaryDirectory() as temp_dir: temp = Path(temp_dir) first = self._model() @@ -798,6 +800,7 @@ def test_preflight_pins_snapshot_and_records_native_mtp_models(self) -> None: ) model.parent.mkdir(parents=True) def gguf_string(value: str) -> bytes: + """Encode UTF-8 text using the GGUF length-prefixed representation.""" encoded = value.encode("utf-8") return struct.pack(" None: self.assertIn("must require exactly the three core lanes", result.stderr) def test_workload_smoke_profile_cannot_claim_an_oracle(self) -> None: + """Prevent smoke profiles from claiming independent-oracle evidence.""" manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) manifest["policy"]["profiles"]["workload-smoke"]["oracle"] = "local-monolithic" with tempfile.TemporaryDirectory() as temp_dir: @@ -289,6 +290,7 @@ def test_workload_smoke_profile_cannot_claim_an_oracle(self) -> None: self.assertIn("workload-smoke must remain provisional and oracle-free", result.stderr) def test_workload_oracle_profile_cannot_drop_the_oracle_lane(self) -> None: + """Require the oracle lane for profiles claiming oracle validation.""" manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) manifest["policy"]["profiles"]["workload-oracle"]["required_lanes"] = ["class-specific-smoke"] with tempfile.TemporaryDirectory() as temp_dir: @@ -299,6 +301,7 @@ def test_workload_oracle_profile_cannot_drop_the_oracle_lane(self) -> None: self.assertIn("workload-oracle requires certified local-monolithic", result.stderr) def test_certified_workload_requires_fixture_and_comparison_evidence(self) -> None: + """Require fixtures and comparison evidence before workload certification.""" manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) model = next(row for row in manifest["models"] if row["profile"] == "workload-oracle") del model["evidence"] @@ -310,6 +313,7 @@ def test_certified_workload_requires_fixture_and_comparison_evidence(self) -> No self.assertIn("evidence must be an object", result.stderr) def test_supplied_plan_rejects_tampered_selected_model_rows(self) -> None: + """Reject changes to canonical model selection or shard metadata in supplied plans.""" generated = self._run(MANIFEST, "--cadence", "manual-full", "--shard-count", "2") self.assertEqual(0, generated.returncode, generated.stderr) plan = json.loads(generated.stdout) @@ -326,6 +330,7 @@ def test_supplied_plan_rejects_tampered_selected_model_rows(self) -> None: self.assertIn("differs from the canonical manifest and selection", result.stderr) def test_supplied_plan_rejects_inflated_oracle_status(self) -> None: + """Reject plans promoting uncertified manifest entries to oracle-certified status.""" generated = self._run(MANIFEST, "--families", "nomic-bert-embedding") self.assertEqual(0, generated.returncode, generated.stderr) plan = json.loads(generated.stdout) @@ -338,6 +343,7 @@ def test_supplied_plan_rejects_inflated_oracle_status(self) -> None: self.assertIn("differs from the canonical manifest and selection", result.stderr) def test_non_chat_model_cannot_claim_the_certified_full_profile(self) -> None: + """Prevent non-chat workloads from inheriting causal-only certification.""" manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) model = next(row for row in manifest["models"] if row["class"] == "embedding") model["profile"] = "full" @@ -359,6 +365,7 @@ def test_duplicate_family_is_rejected(self) -> None: self.assertIn("duplicate family", result.stderr) def test_model_class_is_required_and_selects_class_specific_lanes(self) -> None: + """Require an explicit supported model class and its corresponding validation lanes.""" source = json.loads(MANIFEST.read_text(encoding="utf-8")) model = copy.deepcopy(source["models"][0]) source["models"] = [model] @@ -381,6 +388,7 @@ def test_model_class_is_required_and_selects_class_specific_lanes(self) -> None: self.assertEqual(["embedding-smoke"], selected["certification_lanes"]) def test_projector_classes_require_an_explicit_projector_artifact(self) -> None: + """Require a pinned projector for model classes that consume media.""" source = json.loads(MANIFEST.read_text(encoding="utf-8")) model = copy.deepcopy(source["models"][0]) source["models"] = [model] @@ -396,6 +404,7 @@ def test_projector_classes_require_an_explicit_projector_artifact(self) -> None: self.assertIn("requires an mmproj_artifact", result.stderr) def test_non_causal_classes_reject_split_and_speculative_policy(self) -> None: + """Reject split and speculative policies unsupported by non-causal workloads.""" source = json.loads(MANIFEST.read_text(encoding="utf-8")) model = copy.deepcopy(source["models"][0]) source["models"] = [model] @@ -416,6 +425,7 @@ def test_non_causal_classes_reject_split_and_speculative_policy(self) -> None: self.assertIn("must not request split or MTP certification", split.stderr) def test_inspect_gguf_reports_canonical_dimensions_without_a_manifest(self) -> None: + """Read canonical GGUF dimensions independently of family-manifest metadata.""" with tempfile.TemporaryDirectory() as temp_dir: path = Path(temp_dir) / "fixture.gguf" self._write_gguf(path, 7, 1536) @@ -434,6 +444,7 @@ def test_inspect_gguf_reports_canonical_dimensions_without_a_manifest(self) -> N ) def test_unknown_model_class_is_rejected(self) -> None: + """Reject unknown model classes instead of selecting a causal fallback.""" source = json.loads(MANIFEST.read_text(encoding="utf-8")) source["models"] = [copy.deepcopy(source["models"][0])] source["models"][0]["class"] = "guessed-from-name" diff --git a/scripts/tests/test_runtime_events_model_cadence.py b/scripts/tests/test_runtime_events_model_cadence.py new file mode 100644 index 0000000000..385a4e888a --- /dev/null +++ b/scripts/tests/test_runtime_events_model_cadence.py @@ -0,0 +1,79 @@ +"""Keep the native runtime-event gate's real fixture executable in PR and main CI.""" + +from __future__ import annotations + +import json +from pathlib import Path +import subprocess +import tempfile +import unittest + +import yaml + + +ROOT = Path(__file__).resolve().parents[2] +RESOLVER = ROOT / "scripts/resolve-test-model-manifest.py" + + +class RuntimeEventModelCadenceTests(unittest.TestCase): + """Resolve the exact model selected by the protected Linux runtime workflow.""" + + def gate_inputs(self) -> dict[str, str]: + """Read the workflow's fixture selector instead of duplicating its manifest.""" + workflow = yaml.safe_load( + (ROOT / ".github/workflows/ci-linux-runtime-slice.yml").read_text() + ) + step = next(step for step in workflow["jobs"]["linux_runtime"]["steps"] + if step.get("id") == "gate_model") + self.assertEqual(step["uses"], "./.github/actions/restore-test-model") + self.assertEqual(step["with"]["model_cadence"], + "${{ (inputs.original_event_name == 'pull_request' || " + "inputs.original_event_name == 'pull_request_target') " + "&& 'pull-request' || 'main' }}") + return step["with"] + + def resolve(self, manifest: Path, artifact: str, cadence: str) -> subprocess.CompletedProcess: + """Exercise the production resolver with the gate's single-file requirement.""" + return subprocess.run( + ["python3", str(RESOLVER), str(manifest), "--artifact-id", artifact, + "--cadence", cadence, "--require-single-file"], + cwd=ROOT, text=True, capture_output=True, check=False, + ) + + def test_native_gate_model_resolves_for_pr_and_main(self) -> None: + """Both event branches must resolve before the native reporter test can run.""" + inputs = self.gate_inputs() + for cadence in ("pull-request", "main"): + with self.subTest(cadence=cadence): + result = self.resolve(ROOT / inputs["model_manifest"], + inputs["model_artifact_id"], cadence) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(result.stdout)["artifact_id"], + inputs["model_artifact_id"]) + + def test_missing_cadence_still_fails_closed(self) -> None: + """Adding gate membership must not turn the resolver into a permissive fallback.""" + inputs = self.gate_inputs() + source = json.loads((ROOT / inputs["model_manifest"]).read_text()) + for cadence in ("pull-request", "main"): + with self.subTest(cadence=cadence), tempfile.TemporaryDirectory() as directory: + manifest = json.loads(json.dumps(source)) + artifact = next(row for row in manifest["artifacts"] + if row["id"] == inputs["model_artifact_id"]) + artifact["cadences"] = [value for value in artifact["cadences"] + if value != cadence] + path = Path(directory) / "manifest.json" + path.write_text(json.dumps(manifest)) + result = self.resolve(path, inputs["model_artifact_id"], cadence) + self.assertEqual(result.returncode, 2) + self.assertIn("is not allowed at cadence", result.stderr) + + def test_native_gate_does_not_expand_family_certification_cadences(self) -> None: + """PR/main model retrieval must not add persistent-runner family executions.""" + family = json.loads((ROOT / "ci/llama-canary/family-certified.json").read_text()) + dense = next(row for row in family["models"] if row["family"] == "qwen3-dense") + self.assertEqual(dense["cadences"], ["llama-bump", "manual-full", "nightly"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_skippy_static_link.py b/scripts/tests/test_skippy_static_link.py index 8b2920df50..8938d4bd25 100644 --- a/scripts/tests/test_skippy_static_link.py +++ b/scripts/tests/test_skippy_static_link.py @@ -29,6 +29,7 @@ class SkippyStaticLinkTests(unittest.TestCase): @classmethod def setUpClass(cls) -> None: + """Compile the real build script once for isolated backend-selection tests.""" cls.binary_dir = tempfile.TemporaryDirectory() cls.addClassCleanup(cls.binary_dir.cleanup) cls.binary = Path(cls.binary_dir.name) / "build-script" @@ -43,11 +44,13 @@ def setUpClass(cls) -> None: raise RuntimeError(f"build script fixture failed: {result.stderr}") def _run(self, backend: str, flags: dict[str, str]) -> subprocess.CompletedProcess[str]: + """Execute a backend-selection fixture with Unix-style CMake cache lines.""" return self._run_with_newline(backend, flags, "\n") def _run_with_newline( self, backend: str, flags: dict[str, str], newline: str ) -> subprocess.CompletedProcess[str]: + """Run the build script with isolated archives and controlled CMake cache line endings.""" fixture = tempfile.TemporaryDirectory() self.addCleanup(fixture.cleanup) build_dir = Path(fixture.name) / "native" @@ -78,6 +81,7 @@ def _run_with_newline( ) def test_cpu_ignores_stale_gpu_and_blas_archives(self) -> None: + """Keep CPU linking independent of stale accelerator archives.""" result = self._run("cpu", { "GGML_BLAS": "OFF", "GGML_CUDA": "OFF", "GGML_HIP": "OFF", "GGML_VULKAN": "OFF", "GGML_METAL": "OFF", @@ -89,6 +93,7 @@ def test_cpu_ignores_stale_gpu_and_blas_archives(self) -> None: self.assertNotIn(f"cargo:rustc-link-lib=framework={framework}", result.stdout) def test_active_metal_backend_links_only_cache_enabled_archive(self) -> None: + """Link only the selected Metal archive enabled by CMake.""" result = self._run("metal", {"GGML_METAL": "ON", "GGML_BLAS": "ON"}) self.assertEqual(0, result.returncode, result.stderr) self.assertIn("cargo:rustc-link-lib=static=ggml-metal", result.stdout) @@ -97,21 +102,25 @@ def test_active_metal_backend_links_only_cache_enabled_archive(self) -> None: self.assertNotIn("cargo:rustc-link-lib=static=ggml-cuda", result.stdout) def test_backend_cache_mismatch_fails_closed_despite_stale_archive(self) -> None: + """Reject a disabled selected backend even when a stale archive exists.""" result = self._run("metal", {"GGML_METAL": "OFF"}) self.assertNotEqual(0, result.returncode) self.assertIn("selected backend requires GGML_METAL=ON", result.stderr) def test_unselected_backend_cache_mismatch_fails_closed(self) -> None: + """Reject an accelerator enabled outside the selected native backend.""" result = self._run("cpu", {"GGML_CUDA": "ON"}) self.assertNotEqual(0, result.returncode) self.assertIn("staged backend mismatch: GGML_CUDA=ON", result.stderr) def test_crlf_cache_values_are_recognized(self) -> None: + """Recognize CMake booleans written with Windows line endings.""" result = self._run_with_newline("metal", {"GGML_METAL": "ON"}, "\r\n") self.assertEqual(0, result.returncode, result.stderr) self.assertIn("cargo:rustc-link-lib=static=ggml-metal", result.stdout) def test_enabled_staged_accelerator_requires_matching_selected_backend(self) -> None: + """Require the runtime backend to match enabled staged accelerators.""" for key in ("GGML_CUDA", "GGML_HIP", "GGML_VULKAN", "GGML_METAL"): with self.subTest(key=key): result = self._run("cpu", {key: "ON"}) @@ -119,6 +128,7 @@ def test_enabled_staged_accelerator_requires_matching_selected_backend(self) -> self.assertIn(f"staged backend mismatch: {key}=ON", result.stderr) def test_cache_boolean_accepts_crlf_and_surrounding_whitespace(self) -> None: + """Normalize line endings and whitespace when reading CMake booleans.""" for value in ("ON\r", " TRUE \r", " 1 "): with self.subTest(value=value): result = self._run("metal", {"GGML_METAL": value}) diff --git a/scripts/tests/test_skippy_tts_oracle.py b/scripts/tests/test_skippy_tts_oracle.py index 217e08c8da..3c9dd87e51 100644 --- a/scripts/tests/test_skippy_tts_oracle.py +++ b/scripts/tests/test_skippy_tts_oracle.py @@ -83,6 +83,7 @@ def test_oracle_invocation_matches_candidate_no_repack_context(self) -> None: commands: list[list[str]] = [] def fake_run_logged(command: list[str], _log_path: Path, **_kwargs: object) -> None: + """Emulate independent TTS execution while preserving its logged-output contract.""" commands.append(command) name = "tts-candidate.wav" if command[0] == "cargo" else "tts-monolithic-oracle.wav" write_wav(work_dir / name, [1000, -1000] * 800) From d99df26478671f530397880a30a1254ed81dd795 Mon Sep 17 00:00:00 2001 From: scama Date: Tue, 15 Sep 2026 15:57:49 +1000 Subject: [PATCH 14/18] fix(skippy): initialize disabled prefill token output --- ...ate-sampling-across-all-execution-bounda.patch | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/third_party/llama.cpp/patches/0042-skippy-validate-sampling-across-all-execution-bounda.patch b/third_party/llama.cpp/patches/0042-skippy-validate-sampling-across-all-execution-bounda.patch index 170ad0f797..f925aa4f71 100644 --- a/third_party/llama.cpp/patches/0042-skippy-validate-sampling-across-all-execution-bounda.patch +++ b/third_party/llama.cpp/patches/0042-skippy-validate-sampling-across-all-execution-bounda.patch @@ -94,18 +94,21 @@ index 357222c13..967a6b1ed 100644 } } } -@@ -370,9 +374,12 @@ static enum skippy_status skippy_prefill_chunk_frame_impl( +@@ -370,9 +374,15 @@ static enum skippy_status skippy_prefill_chunk_frame_impl( return status; } - if (out_predicted_token != nullptr) { - *out_predicted_token = session->stage_model->config.include_output ? - skippy_sample_token(session, sampling) : -1; -+ if (out_predicted_token != nullptr && session->stage_model->config.include_output) { -+ status = skippy_store_sampled_token( -+ skippy_sample_token(session, sampling), out_predicted_token, out_error); -+ if (status != SKIPPY_STATUS_OK) { -+ return status; ++ if (out_predicted_token != nullptr) { ++ *out_predicted_token = -1; ++ if (session->stage_model->config.include_output) { ++ status = skippy_store_sampled_token( ++ skippy_sample_token(session, sampling), out_predicted_token, out_error); ++ if (status != SKIPPY_STATUS_OK) { ++ return status; ++ } + } } From da63397e5134f396f178b131525145b64a13ec46 Mon Sep 17 00:00:00 2001 From: scama Date: Tue, 15 Sep 2026 16:11:50 +1000 Subject: [PATCH 15/18] chore(skippy): regenerate certified split roster after main sync --- .../src/inference/skippy/split-certified.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/mesh-llm-host-runtime/src/inference/skippy/split-certified.json b/crates/mesh-llm-host-runtime/src/inference/skippy/split-certified.json index 5b84f942bb..f15d2f4728 100644 --- a/crates/mesh-llm-host-runtime/src/inference/skippy/split-certified.json +++ b/crates/mesh-llm-host-runtime/src/inference/skippy/split-certified.json @@ -3,7 +3,7 @@ "native_recipe": { "llama_upstream_sha": "3057bb66c86c46d5781e50e85462a760ba7d1feb", "skippy_abi": "0.1.55", - "patch_queue_sha256": "34b271ab292394ecea3171a7bb4a6aac2bc5889eb5fc88b3a025dbf823f33e1a" + "patch_queue_sha256": "f7f52d1f145c3f8cd6fcd05667a9d50d1d04f6585a5b238e6e59a8de39a75814" }, "models": [ { From a0ddd1a4dfaba3d7e419ef0a36e5c2f58c9f112e Mon Sep 17 00:00:00 2001 From: scama Date: Tue, 15 Sep 2026 16:51:06 +1000 Subject: [PATCH 16/18] build(skippy): lock non-chat workload dependencies --- Cargo.lock | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index b33c252041..1916ae7424 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -558,6 +558,7 @@ dependencies = [ "matchit", "memchr", "mime", + "multer", "percent-encoding", "pin-project-lite", "serde_core", @@ -4398,6 +4399,23 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fafa6961cabd9c63bcd77a45d7e3b7f3b552b70417831fb0f56db717e72407e" +[[package]] +name = "multer" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" +dependencies = [ + "bytes", + "encoding_rs", + "futures-util", + "http", + "httparse", + "memchr", + "mime", + "spin 0.9.9", + "version_check", +] + [[package]] name = "multimap" version = "0.10.1" @@ -5174,6 +5192,7 @@ version = "0.76.1" dependencies = [ "async-trait", "axum", + "base64 0.22.1", "futures-core", "futures-util", "http-body-util", From cbe14fe915e4903fde0c397726813086cea012b4 Mon Sep 17 00:00:00 2001 From: IvGolovach <20299097+IvGolovach@users.noreply.github.com> Date: Tue, 15 Sep 2026 12:18:24 -0700 Subject: [PATCH 17/18] fix(ci): preserve workload classes in battery summaries Resolve preflight model classes from the validated policy instead of labeling all rows as causal generation. Keep environment checks unclassified and preserve certification records and reconciliation unchanged. Validation: shell syntax and Python compilation passed; four workload lane tests passed; just ci-validate passed with 1601 Python tests and 9 skipped. --- scripts/skippy-family-battery.sh | 17 +++++++--- scripts/tests/test_workload_lane_execution.py | 31 +++++++++++++++++++ 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/scripts/skippy-family-battery.sh b/scripts/skippy-family-battery.sh index a09b7c1fe4..ec8e9ddd42 100755 --- a/scripts/skippy-family-battery.sh +++ b/scripts/skippy-family-battery.sh @@ -1007,6 +1007,17 @@ run_resolved_manifest() { done < "$resolved_manifest" } +write_lane_summary() { + jq -sr --slurpfile policy "$POLICY_PLAN_COPY" ' + def model_class($row): + $row.workload_class // + ([$policy[0].selected_models[] | select(.family == $row.family) | .class][0] // ""); + ["family","class","split_layer","lane","status","outcome","exit_code"], + (.[] as $row | $row.outcomes[] | [$row.family,model_class($row),($row.split_layer // ""),.name,.status,.outcome,.exit_code]) + | @tsv + ' "$RESULTS_JSONL" > "$SUMMARY_TSV" +} + build_certification_binaries if ! preflight_manifest "$POLICY_PLAN_COPY"; then echo "family battery preflight failed; no certification lane was started" >&2 @@ -1036,11 +1047,7 @@ fi echo if (( DRY_RUN == 0 )); then - jq -sr ' - ["family","class","split_layer","lane","status","outcome","exit_code"], - (.[] as $row | $row.outcomes[] | [$row.family,($row.workload_class // "causal_generation"),($row.split_layer // ""),.name,.status,.outcome,.exit_code]) - | @tsv - ' "$RESULTS_JSONL" > "$SUMMARY_TSV" + write_lane_summary { echo "# Supported-families battery" echo diff --git a/scripts/tests/test_workload_lane_execution.py b/scripts/tests/test_workload_lane_execution.py index 5982b4477f..a179335967 100644 --- a/scripts/tests/test_workload_lane_execution.py +++ b/scripts/tests/test_workload_lane_execution.py @@ -22,6 +22,37 @@ def shell_function(script: str, name: str) -> str: class WorkloadLaneExecutionTests(unittest.TestCase): + def test_summary_preserves_preflight_classes_without_claiming_certification(self) -> None: + """Use planned classes for preflight; environment checks have no model class.""" + classes = ["causal_generation", "embedding", "rerank", "encoder_decoder", + "ocr", "speech_synthesis", "speech_recognition"] + models = [{"family": f"family-{index}", "class": value} + for index, value in enumerate(classes)] + outcome = {"name": "model-preflight", "status": "pass", "outcome": "pass", "exit_code": 0} + rows = [{"family": model["family"], "outcomes": [outcome]} for model in models] + rows.extend([ + {"family": "battery", "outcomes": [{**outcome, "name": "environment-preflight"}]}, + {"family": "family-0", "split_layer": 2, "outcomes": [{**outcome, "name": "chain"}]}, + {"family": "explicit", "workload_class": "embedding", + "outcomes": [{**outcome, "name": "embedding-oracle"}]}, + ]) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + policy, results, summary = (root / name for name in ("plan.json", "results.jsonl", "summary.tsv")) + policy.write_text(json.dumps({"selected_models": models})) + source = "\n".join(json.dumps(row) for row in rows) + results.write_text(source) + env = {**os.environ, "POLICY_PLAN_COPY": str(policy), + "RESULTS_JSONL": str(results), "SUMMARY_TSV": str(summary)} + script = "set -euo pipefail\n" + shell_function("skippy-family-battery.sh", "write_lane_summary") + result = subprocess.run(["bash", "-c", script + "\nwrite_lane_summary"], env=env, + capture_output=True, text=True, check=False, timeout=15) + self.assertEqual(0, result.returncode, result.stderr) + actual = [line.split("\t") for line in summary.read_text().splitlines()] + self.assertEqual(["family", "class", "split_layer", "lane", "status", "outcome", "exit_code"], actual[0]) + self.assertEqual(classes + ["", "causal_generation", "embedding"], [row[1] for row in actual[1:]]) + self.assertEqual(source, results.read_text(), "summary must not promote preflight rows to certifications") + def run_lane(self, dry_run: bool) -> tuple[subprocess.CompletedProcess[str], list[dict]]: """Run an isolated battery function with deterministic producer and log fixtures.""" with tempfile.TemporaryDirectory() as directory: From 308a1b502e850acdd2072d5b46d221651a039e2e Mon Sep 17 00:00:00 2001 From: IvGolovach <20299097+IvGolovach@users.noreply.github.com> Date: Tue, 15 Sep 2026 12:42:49 -0700 Subject: [PATCH 18/18] fix(ci): defer cancellation until subprocess waits unwind Install cancellation handlers before spawning and only record the first signal. Perform process-group cleanup after wait returns, preserving child exit status and restoring the original handlers. Add deterministic spawn/wait boundary regressions and guarantee integration-fixture cleanup on failures. Validation: 37 related tests passed; 200 real SIGINT/SIGTERM cancellations passed; child exit statuses 0, 7 and 125 were preserved; just ci-validate passed with 1603 Python tests and 9 skipped. --- scripts/run-command-with-timeout.py | 67 ++++++++----- .../test_llama_upstream_canary_contract.py | 34 ++++--- .../tests/test_run_command_with_timeout.py | 96 +++++++++++++++++++ 3 files changed, 160 insertions(+), 37 deletions(-) create mode 100644 scripts/tests/test_run_command_with_timeout.py diff --git a/scripts/run-command-with-timeout.py b/scripts/run-command-with-timeout.py index ee6a98181c..c7a61d981c 100755 --- a/scripts/run-command-with-timeout.py +++ b/scripts/run-command-with-timeout.py @@ -8,6 +8,7 @@ import signal import subprocess import sys +import time def parse_args() -> argparse.Namespace: @@ -41,39 +42,53 @@ def terminate_group(process: subprocess.Popen[bytes]) -> None: def main() -> int: args = parse_args() - # The wrapper is commonly invoked from manifest-reading shell loops. A - # child must never inherit and consume the loop's stdin, because doing so - # can silently drop later planned rows. Commands in this harness are fully - # argument-driven, so EOF is the only valid stdin contract. - process = subprocess.Popen( - args.command, - stdin=subprocess.DEVNULL, - start_new_session=True, - ) + received_signal: int | None = None - def terminate_on_signal(signum: int, _frame: object) -> None: - # Prevent a second cancellation signal from interrupting cleanup and - # leaving descendants behind on the persistent runner. - signal.signal(signal.SIGINT, signal.SIG_IGN) - signal.signal(signal.SIGTERM, signal.SIG_IGN) - print( - f"{args.label} received signal {signum}; terminating process group", - file=sys.stderr, - ) - terminate_group(process) - raise SystemExit(128 + signum) + def request_termination(signum: int, _frame: object) -> None: + # A handler can interrupt Popen construction or wait's internal lock. + # Only record intent here; never wait, print, or clean up reentrantly. + nonlocal received_signal + if received_signal is None: + received_signal = signum - signal.signal(signal.SIGINT, terminate_on_signal) - signal.signal(signal.SIGTERM, terminate_on_signal) + previous_handlers = { + signum: signal.signal(signum, request_termination) + for signum in (signal.SIGINT, signal.SIGTERM) + } try: - return process.wait(timeout=args.seconds) - except subprocess.TimeoutExpired: + # Commands are argument-driven: inheriting a manifest loop's stdin + # could silently consume later planned rows, so children receive EOF. + process = subprocess.Popen( + args.command, + stdin=subprocess.DEVNULL, + start_new_session=True, + ) + deadline = time.monotonic() + args.seconds + while received_signal is None: + remaining = deadline - time.monotonic() + if remaining <= 0: + print( + f"{args.label} timed out after {args.seconds}s; terminating process group", + file=sys.stderr, + ) + terminate_group(process) + return 124 + try: + returncode = process.wait(timeout=min(0.1, remaining)) + except subprocess.TimeoutExpired: + continue + if received_signal is None: + return returncode + print( - f"{args.label} timed out after {args.seconds}s; terminating process group", + f"{args.label} received signal {received_signal}; terminating process group", file=sys.stderr, ) terminate_group(process) - return 124 + return 128 + received_signal + finally: + for signum, handler in previous_handlers.items(): + signal.signal(signum, handler) if __name__ == "__main__": diff --git a/scripts/tests/test_llama_upstream_canary_contract.py b/scripts/tests/test_llama_upstream_canary_contract.py index 48ed9b66ea..5b43ce2de7 100644 --- a/scripts/tests/test_llama_upstream_canary_contract.py +++ b/scripts/tests/test_llama_upstream_canary_contract.py @@ -544,17 +544,29 @@ def test_timeout_runner_cleans_process_group_when_signalled(self) -> None: stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) - assert wrapper.stdout is not None - child_pid = int(wrapper.stdout.readline()) - wrapper.send_signal(received_signal) - _, stderr = wrapper.communicate(timeout=15) - - self.assertEqual(128 + received_signal, wrapper.returncode) - self.assertIn( - f"signal-fixture received signal {received_signal}", stderr - ) - with self.assertRaises(ProcessLookupError): - os.kill(child_pid, 0) + child_pid = None + try: + assert wrapper.stdout is not None + child_pid = int(wrapper.stdout.readline()) + wrapper.send_signal(received_signal) + _, stderr = wrapper.communicate(timeout=15) + + self.assertEqual(128 + received_signal, wrapper.returncode) + self.assertIn( + f"signal-fixture received signal {received_signal}", stderr + ) + with self.assertRaises(ProcessLookupError): + os.kill(child_pid, 0) + finally: + # A regression must not leave its fixture running on CI. + if wrapper.poll() is None: + wrapper.kill() + if child_pid is not None: + try: + os.killpg(child_pid, signal.SIGKILL) + except ProcessLookupError: + pass + wrapper.communicate(timeout=5) def test_timeout_runner_closes_manifest_stdin_for_children(self) -> None: result = subprocess.run( diff --git a/scripts/tests/test_run_command_with_timeout.py b/scripts/tests/test_run_command_with_timeout.py new file mode 100644 index 0000000000..e78dd65566 --- /dev/null +++ b/scripts/tests/test_run_command_with_timeout.py @@ -0,0 +1,96 @@ +"""Deterministic signal-boundary regressions for the canary process supervisor.""" + +from __future__ import annotations + +import argparse +import importlib.util +from pathlib import Path +import signal +import subprocess +import unittest +from unittest import mock + + +SOURCE = Path(__file__).resolve().parents[1] / "run-command-with-timeout.py" +SPEC = importlib.util.spec_from_file_location("timeout_runner", SOURCE) +assert SPEC is not None and SPEC.loader is not None +RUNNER = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(RUNNER) + + +class TimeoutSignalSafetyTests(unittest.TestCase): + def exercise_signal_boundary(self, boundary: str, signum: int) -> None: + """Inject cancellation at a boundary without sending signals to the test runner.""" + handlers = {signal.SIGINT: signal.SIG_DFL, signal.SIGTERM: signal.SIG_DFL} + events: list[str] = [] + in_spawn = False + in_wait = False + process = mock.Mock() + + def install_handler(number, handler): + previous = handlers[number] + handlers[number] = handler + return previous + + def request_cancel(number): + self.assertTrue(callable(handlers[number]), "handler must be installed before spawn") + handlers[number](number, None) + + def spawn(*_args, **_kwargs): + nonlocal in_spawn + in_spawn = True + try: + if boundary == "spawn": + request_cancel(signum) + finally: + in_spawn = False + events.append("spawned") + return process + + def wait(*_args, **kwargs): + nonlocal in_wait + in_wait = True + try: + request_cancel(signum) + raise subprocess.TimeoutExpired("fixture", kwargs["timeout"]) + finally: + in_wait = False + + def cleanup(actual_process): + self.assertIs(process, actual_process) + self.assertFalse(in_spawn, "cleanup needs the completed Popen object") + self.assertFalse(in_wait, "cleanup must not reenter Popen.wait from a signal handler") + # Repeated cancellation during cleanup must neither reenter cleanup + # nor replace the signal that determined the wrapper's exit status. + request_cancel(signal.SIGINT if signum == signal.SIGTERM else signal.SIGTERM) + events.append("cleaned") + + process.wait.side_effect = wait + args = argparse.Namespace(seconds=30, label="fixture", command=["fixture"]) + with ( + mock.patch.object(RUNNER, "parse_args", return_value=args), + mock.patch.object(RUNNER.signal, "signal", side_effect=install_handler), + mock.patch.object(RUNNER.subprocess, "Popen", side_effect=spawn) as popen, + mock.patch.object(RUNNER, "terminate_group", side_effect=cleanup) as terminate, + ): + self.assertEqual(128 + signum, RUNNER.main()) + popen.assert_called_once_with( + ["fixture"], stdin=subprocess.DEVNULL, start_new_session=True, + ) + terminate.assert_called_once_with(process) + self.assertEqual(["spawned", "cleaned"], events) + self.assertEqual({signal.SIGINT: signal.SIG_DFL, signal.SIGTERM: signal.SIG_DFL}, handlers) + + def test_cancellation_during_spawn_waits_for_child_ownership(self) -> None: + for signum in (signal.SIGINT, signal.SIGTERM): + with self.subTest(signum=signum): + self.exercise_signal_boundary("spawn", signum) + + def test_cancellation_during_wait_defers_cleanup_until_wait_unwinds(self) -> None: + for signum in (signal.SIGINT, signal.SIGTERM): + with self.subTest(signum=signum): + self.exercise_signal_boundary("wait", signum) + + +if __name__ == "__main__": + unittest.main()