diff --git a/.agents/skills/manage-ci/references/current-inventory.md b/.agents/skills/manage-ci/references/current-inventory.md index 470f62f9c0..e48542c9dd 100644 --- a/.agents/skills/manage-ci/references/current-inventory.md +++ b/.agents/skills/manage-ci/references/current-inventory.md @@ -49,10 +49,20 @@ caching). The runner preflight prepends `/Users/lab/.local/bin` and executes `goose --version`, so a missing, damaged, or non-executable agent binary fails before the changed-pin harness starts. Before native compilation, `scripts/plan-family-battery.py` validates the versioned JSON family policy, -the mandatory three-lane contract for every certified profile, and every exact -artifact revision/file in the immutable local cache. It reads only GGUF -metadata headers, requires each artifact to have at least one metadata-bearing -shard, and requires every shard that carries `*.block_count` and +the three core parity lanes for certified causal rows, a class-specific +smoke plus independent local-monolithic oracle pair for each of the six registry-generated non-chat rows +(`embedding`, `rerank`, `encoder_decoder`, `ocr`, `speech_synthesis`, and +`speech_recognition`): respectively `embedding-smoke`, +`rerank-smoke`, `encoder-decoder-smoke`, `ocr-smoke`, +`speech-synthesis-smoke`, and `speech-recognition-smoke`, each paired with its +`-oracle` lane. These lanes exercise local full-model and HTTP behavior and +independent equivalence. Workload readiness uses the planned per-model deadline +for both servers; embedding certification requires the official Python SDK smoke. +Dry-run planning needs no oracle tools; a missing execution prerequisite records +failed lanes and does not discard later family results. It also +checks every exact artifact revision/file in the immutable local cache. It +reads only GGUF metadata headers, requires each artifact to have at least one +metadata-bearing shard, and requires every shard that carries `*.block_count` and `*.embedding_length` to equal the planned runtime range and activation width before compilation; Qwen4 experimental artifacts derive their wider boundary from `hyper_connection.count * embedding_length`. It emits @@ -68,16 +78,23 @@ local disk and moves each repo to NFS). On Apple Silicon, the wrapper and generated-family rewriter re-exec as native arm64 before creating build state; the rewriter discards a CMake cache for any other architecture. Correctness lanes derive filtered-load resident tensor names from the native stage graph -planner, including GGUFs with non-finite metadata values. The workflow builds its five +planner, including GGUFs with non-finite metadata values. The workflow builds the certification binaries before the manifest lanes; the family battery builds -them once itself unless `--skip-build` is selected, in which case it verifies -that every binary already exists. Before any certification starts, every GGUF is resolved +its four required executables once unless `--skip-build` is selected, in which case it verifies +that every binary already exists. Scheduled, changed-pin, and forced runs +use the complete 83-target causal roster plus six non-chat workload rows. +Both ordinary and independent changed-pin verification build a run-scoped +CPU oracle closure with `just skippy-workload-oracles-build`. Generated +`SKIPPY_WORKLOAD_*` paths select separate reference and candidate binaries +without replacing Metal outputs; source- and executable-bound +`producer.json` is verified before consumption. Non-chat rows require +class-specific smoke and oracle evidence and never certify staged splits. Before any certification starts, every GGUF is resolved directly by the immutable snapshot SHA checked into `ci/llama-canary/family-certified.json`. The runtime preflight records the revisions, verifies all shard/tensor scans and declared runtime/MTP layer -counts/model bytes, then uses the production topology capability rules to +counts/model bytes, then uses, for causal rows, the production topology capability rules to classify every interior boundary and select accepted balanced two-stage and -three-stage cuts. Each model runs one consolidated live certification and +three-stage cuts. Each causal model runs one consolidated live certification and shares one unloaded monolithic token oracle between its single-step and chain lanes. Preflight also checks disk headroom and certification ports. Native MTP/NextN heads remain part of the single target model; the battery never reopens that model as a separate draft. diff --git a/.github/workflows/llama-upstream-canary.yml b/.github/workflows/llama-upstream-canary.yml index 4db7b69ae7..4fa61bad86 100644 --- a/.github/workflows/llama-upstream-canary.yml +++ b/.github/workflows/llama-upstream-canary.yml @@ -18,7 +18,7 @@ on: # It must never execute a caller-selected ref on the persistent # `family-certify` self-hosted runner. The job token is read-only. Changed pins # are repaired by one agent, independently verified by the harness, and then -# published from a fresh GitHub-hosted job; unchanged nightly sentinels remain +# published from a fresh GitHub-hosted job; unchanged full-roster checks remain # read-only. permissions: contents: read @@ -334,6 +334,14 @@ jobs: WORK_DIR: ${{ runner.temp }}/skippy-ci-smoke run: scripts/skippy-ci-smoke.sh + - name: Build pinned CPU workload oracles and candidate + if: steps.prepare.outcome == 'success' && steps.sha.outputs.certify == 'true' && steps.family_patch.outcome == 'success' + run: | + set -euo pipefail + workload_root="${LLAMA_STAGE_BUILD_DIR:?}-workloads" + arch -arm64 just skippy-workload-oracles-build "$workload_root" + bash scripts/skippy-workload-oracles-build.sh --print-env "$workload_root" >> "$GITHUB_ENV" + - name: Supported-families certification battery (parity gate) id: battery if: steps.prepare.outcome == 'success' && steps.sha.outputs.certify == 'true' && steps.family_patch.outcome == 'success' @@ -361,6 +369,7 @@ jobs: name: llama-family-battery-${{ github.run_id }}-${{ github.run_attempt }} path: | target/family-battery/${{ github.run_id }}-${{ github.run_attempt }}/ + ${{ env.LLAMA_STAGE_BUILD_DIR }}-workloads/producer.json target/skippy-stage-rewriter-check/ if-no-files-found: warn retention-days: 14 @@ -406,9 +415,9 @@ jobs: && steps.family_patch.outcome == 'success' run: | { - echo "## llama.cpp nightly family sentinels" + echo "## llama.cpp nightly full family certification" echo - echo "The pinned upstream passed the dense, hybrid/composite, and recurrent sentinel battery." + echo "The pinned upstream passed the full causal split and non-chat workload battery." echo echo "Certified families: ${{ steps.family_plan.outputs.family_count }}" cat /tmp/llama-upstream-pr.md diff --git a/Cargo.lock b/Cargo.lock index b33c252041..1916ae7424 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -558,6 +558,7 @@ dependencies = [ "matchit", "memchr", "mime", + "multer", "percent-encoding", "pin-project-lite", "serde_core", @@ -4398,6 +4399,23 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fafa6961cabd9c63bcd77a45d7e3b7f3b552b70417831fb0f56db717e72407e" +[[package]] +name = "multer" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" +dependencies = [ + "bytes", + "encoding_rs", + "futures-util", + "http", + "httparse", + "memchr", + "mime", + "spin 0.9.9", + "version_check", +] + [[package]] name = "multimap" version = "0.10.1" @@ -5174,6 +5192,7 @@ version = "0.76.1" dependencies = [ "async-trait", "axum", + "base64 0.22.1", "futures-core", "futures-util", "http-body-util", diff --git a/ci/ci.md b/ci/ci.md index c549f84d48..ea4e684178 100644 --- a/ci/ci.md +++ b/ci/ci.md @@ -26,7 +26,7 @@ and acceptance criteria are in `.omo/specs/pr-ci-optimization.md`. | `nightly-stability.yml` / `nightly-stability-run.yml` | daily schedule, dispatch / reusable | GitHub-hosted live-endpoint evidence. The general stability and KV tool-loop/prefix-reuse harnesses run independently, upload both evidence sets, and preserve either failure. The reusable workflow accepts no runner label. | | `nightly-kv-coverage.yml` | daily schedule, dispatch | Trusted-`main`, read-only, GitHub-hosted expansion of deterministic radix lease/eviction and blob-ownership state machines. Seed/step budgets and the exact source SHA are uploaded; no secrets or privileged runner are used. | | `agentic-replay-nightly.yml` | daily schedule, trusted-main dispatch | Coding-agent replay benchmark on the persistent macOS `micstudio` runner. The fixed `[self-hosted, X64, macOS, family-certify, agentic-replay]` selector is backed by a fail-closed `RUNNER_NAME=micstudio` check; scheduled and manual execution is restricted to trusted `main`. It resolves exact model and trajectory revisions from the pre-warmed Hugging Face cache, verifies their SHA-256 digests, cross-checks the trajectory pin against the canonical harness before download, derives replay shape from `ci/agentic-replay-nightly/matrix.json`, fails closed on history lookup errors, writes the summary even when the regression gate fails, confines the write-capable HF token to the publish step, uploads immutable evidence, and keeps persistent-runner repair credential-free: the repair loop emits a patch, body and status artifact for its run/attempt, while a separate canonical-main, failed-run GitHub-hosted job validates the artifact, applies the patch with hooks disabled, and uses `CANARY_REPAIR_TOKEN` only to publish a deterministic run/attempt branch and PR. | -| `llama-upstream-canary.yml` | daily schedule, dispatch | Trusted default-branch llama.cpp bump certification on the self-hosted `family-certify` runner. Its toolchain preflight prepends `/Users/lab/.local/bin` and executes `goose --version` before any changed-pin agent work. It never runs as ordinary push or PR CI. Canary runs share a non-cancelling concurrency group, so a scheduled run queues behind active manual or scheduled work instead of discarding the candidate workspace. `scripts/plan-family-battery.py` validates the generated `ci/llama-canary/family-certified.json` policy (sourced from `ci/model-artifacts/registry.json`) and every file's exact immutable cache blob identity and byte size before native compilation. Each target/draft artifact must have at least one metadata-bearing GGUF shard; every shard that carries architecture dimensions must match the declared runtime range and activation width, including Qwen4's `hyper_connection.count * embedding_length` boundary. Optional `mmproj_artifact` rows pin a projector GGUF sidecar (exact blob identity, exempt from trunk-dimension checks), and each family that pins one runs an additional multimodal smoke lane after its core lanes: the real-projector + deterministic-image harness in `crates/skippy-server/src/frontend/tests/multimodal.rs` (local monolithic and split stages) via `SKIPPY_MM_*`, reconciled against the plan like every other lane. It emits deterministic bounded matrix shards and records the plan with evidence. The current single-runner workflow consumes one selected-family shard and builds the certification binaries once. Scheduled, changed-pin, and forced runs all consume the same complete supported-family roster. Before any lane starts, the battery verifies shard/tensor scans, declared runtime/MTP layer counts, model bytes, disk headroom and certification ports. Native MTP/NextN heads remain part of the single target model; the battery does not reopen the model as a separate draft. Those rows require native draft sidebands in staged single-step and chain correctness, where each proposed token is verified against the target. Every certified profile must retain strict `single-step`, `chain`, and `state-handoff` parity. Filtered correctness stages derive their exact resident tensor names from the native stage graph planner, including GGUFs with non-finite metadata values. Single-step and chain exercise the sole shipping raw-f32 activation wire and any mismatch is a hard failure. Planned families, product-planner-selected cuts, and multimodal smokes are reconciled exactly against executed lanes and recorded results. Declared per-model or model-size-derived startup deadlines, complete-certification wall-clock limits and typed lane outcomes are recorded, and immutable plans/model manifests/preflight evidence/certification logs upload even on failure or cancellation. Manual dispatch can force this certification when the upstream SHA is unchanged. Persistent-runner execution is always a read-only checkout of trusted `main`. Changed pins give one agent session the complete developer task: repair or regenerate the queue, address ABI fallout, and iterate through every canonical gate. Both the repair and independent-verifier checkouts configure the same repository-local `mesh-llama-canary-bot` identity before invoking the harness, so candidate commit creation does not depend on persistent-runner global Git configuration. The repair loop has an 11.5-hour deadline while the complete roster runtime is measured, and the agent has no GitHub credentials. Goose uses its native text renderer for readable assistant, thinking, tool, and error output in the live Actions log and retained `agent.log`. A zero exit from the coding process only yields control to the trusted wrapper; the wrapper runs the full candidate gates and returns current failure logs to the same session until those gates pass or the deadline expires. Existing certification and parity rows remain immutable. The agent may only correct `resources.estimated_model_bytes`, which the immutable GGUF scan rechecks, and append classification-only parity rows for source files missing from the manifest; those rows cannot add artifact or certification authority. Only a green repair pass is snapshotted as an unreachable local commit and uploaded as a thin candidate bundle. A separate self-hosted verification job and checkout download that bundle, materialize its commit in a fresh detached worktree, and independently run one ordered `prepare -> manifest-policy -> build -> certify` pass with a 12-hour budget and new native-build and family-evidence directories. Only the exact passing commit is exported as the certified bundle. Before each changed-pin candidate gate, the trusted wrapper regenerates the exact-artifact split certification roster for the candidate recipe; only a complete battery pass is snapshotted, and the independent verifier checks the roster against the llama pin, Skippy ABI, and ordered patch queue before publication. Unchanged-pin certification checks the roster without mutating it. A separate success-gated job on a fresh GitHub-hosted runner receives the repair token, validates the one-day bundle artifact, pushes a run-specific branch, and opens a normal exact-head PR as its final external mutation. Agent or verification failures retain logs and create no branch or PR. Successful publication leaves the canary green; changed pins are never pushed directly to `main`. Unchanged scheduled and forced certifications remain read-only and do not invoke the agent. Runner requires `HF_CACHE=/Users/lab/models/huggingface`, verifies its `hub` directory, exports `HF_HOME` and `HF_HUB_CACHE` from that root, and stays offline on the NFS-backed cache (`HF_HUB_OFFLINE=1`; no `flock`, so the runner never downloads). | +| `llama-upstream-canary.yml` | daily schedule, dispatch | Trusted default-branch llama.cpp bump certification on the self-hosted `family-certify` runner. Its toolchain preflight prepends `/Users/lab/.local/bin` and executes `goose --version` before any changed-pin agent work. It never runs as ordinary push or PR CI. Canary runs share a non-cancelling concurrency group, so a scheduled run queues behind active manual or scheduled work instead of discarding the candidate workspace. `scripts/plan-family-battery.py` validates the generated `ci/llama-canary/family-certified.json` policy (sourced from `ci/model-artifacts/registry.json`) and every file's exact immutable cache blob identity and byte size before native compilation. Each target/draft artifact must have at least one metadata-bearing GGUF shard; every shard that carries architecture dimensions must match the declared runtime range and activation width, including Qwen4's `hyper_connection.count * embedding_length` boundary. Optional `mmproj_artifact` rows pin a projector GGUF sidecar (exact blob identity, exempt from trunk-dimension checks), and each causal family that pins one runs an additional multimodal smoke lane after its core lanes: the real-projector + deterministic-image harness in `crates/skippy-server/src/frontend/tests/multimodal.rs` (local monolithic and split stages) via `SKIPPY_MM_*`, reconciled against the plan like every other lane. It emits deterministic bounded matrix shards and records the plan with evidence. The current single-runner workflow consumes one selected-family shard and builds the certification binaries once. Scheduled, changed-pin, and forced runs all consume the same complete supported-family roster. Before any lane starts, the battery verifies shard/tensor scans, declared runtime/MTP layer counts, model bytes, disk headroom and certification ports. Native MTP/NextN heads remain part of the single target model; the battery does not reopen the model as a separate draft. Those rows require native draft sidebands in staged single-step and chain correctness, where each proposed token is verified against the target. Every certified causal profile must retain strict `single-step`, `chain`, and `state-handoff` parity; non-chat profiles require their separate smoke and oracle lanes. Filtered correctness stages derive their exact resident tensor names from the native stage graph planner, including GGUFs with non-finite metadata values. Single-step and chain exercise the sole shipping raw-f32 activation wire and any mismatch is a hard failure. Planned families, product-planner-selected cuts, and multimodal smokes are reconciled exactly against executed lanes and recorded results. Declared per-model or model-size-derived startup deadlines, complete-certification wall-clock limits and typed lane outcomes are recorded, and immutable plans/model manifests/preflight evidence/certification logs upload even on failure or cancellation. Manual dispatch can force this certification when the upstream SHA is unchanged. Persistent-runner execution is always a read-only checkout of trusted `main`. Changed pins give one agent session the complete developer task: repair or regenerate the queue, address ABI fallout, and iterate through every canonical gate. Both the repair and independent-verifier checkouts configure the same repository-local `mesh-llama-canary-bot` identity before invoking the harness, so candidate commit creation does not depend on persistent-runner global Git configuration. The repair loop has an 11.5-hour deadline while the complete roster runtime is measured, and the agent has no GitHub credentials. Goose uses its native text renderer for readable assistant, thinking, tool, and error output in the live Actions log and retained `agent.log`. A zero exit from the coding process only yields control to the trusted wrapper; the wrapper runs the full candidate gates and returns current failure logs to the same session until those gates pass or the deadline expires. Existing certification and parity rows remain immutable. The agent may only correct `resources.estimated_model_bytes`, which the immutable GGUF scan rechecks, and append classification-only parity rows for source files missing from the manifest; those rows cannot add artifact or certification authority. Only a green repair pass is snapshotted as an unreachable local commit and uploaded as a thin candidate bundle. A separate self-hosted verification job and checkout download that bundle, materialize its commit in a fresh detached worktree, and independently run one ordered `prepare -> manifest-policy -> build -> certify` pass with a 12-hour budget and new native-build and family-evidence directories. Only the exact passing commit is exported as the certified bundle. Before each changed-pin candidate gate, the trusted wrapper regenerates the exact-artifact split certification roster for the candidate recipe; only a complete battery pass is snapshotted, and the independent verifier checks the roster against the llama pin, Skippy ABI, and ordered patch queue before publication. Unchanged-pin certification checks the roster without mutating it. A separate success-gated job on a fresh GitHub-hosted runner receives the repair token, validates the one-day bundle artifact, pushes a run-specific branch, and opens a normal exact-head PR as its final external mutation. Agent or verification failures retain logs and create no branch or PR. Successful publication leaves the canary green; changed pins are never pushed directly to `main`. Unchanged scheduled and forced certifications remain read-only and do not invoke the agent. Runner requires `HF_CACHE=/Users/lab/models/huggingface`, verifies its `hub` directory, exports `HF_HOME` and `HF_HUB_CACHE` from that root, and stays offline on the NFS-backed cache (`HF_HUB_OFFLINE=1`; no `flock`, so the runner never downloads). | The changed-pin canary wrapper owns the target-pin transition: it writes the sole upstream selector, `third_party/llama.cpp/upstream.txt`, before the agent @@ -56,9 +56,25 @@ Evidence uploads explicitly admit success, failure, or cancellation so an operator cancellation still preserves the available repair and verification logs without making publication or status-reporting steps resist cancellation. -Scheduled, changed-pin, and forced certification runs all consume the same -complete supported-family roster. Cadence labels describe why the workflow -ran; they do not filter model coverage. The +Scheduled, changed-pin, and forced certification runs consume the same full +roster: 83 causal split targets and six non-chat workload rows. Cadence labels +describe the trigger, not a model filter. Both ordinary and independently +verified changed-pin canaries build a run-specific CPU oracle closure with +`just skippy-workload-oracles-build`: pinned monolithic server/completion/TTS +references and a separate static CPU candidate, without replacing Metal +outputs. Generated `SKIPPY_WORKLOAD_*` paths replace ambient configuration; +source- and executable-bound `producer.json` is verified before consumption. +Non-chat rows require both class-specific smoke and oracle evidence, not +split certification. Every selected GGUF/projector must exist in the verified +read-only lab cache; provisioning that cache is an external runner operation. + +Dry runs print workload commands without requiring oracle binaries. Executable +runs record missing-oracle failures and continue with the remaining rows. Each +row applies its startup deadline separately to candidate and reference HTTP +servers. Embedding certification also requires the official Python SDK smoke; +a missing SDK is a failure, not a skipped passing check. + +The competitive benchmark can optionally download exact-cohort history from `MESH_PERFORMANCE_HISTORY_DATASET`, validate the checked-in schema, report regression candidates, and append one immutable run shard using diff --git a/ci/llama-canary/family-certified.json b/ci/llama-canary/family-certified.json index b9700f461b..d38f602767 100644 --- a/ci/llama-canary/family-certified.json +++ b/ci/llama-canary/family-certified.json @@ -16,12 +16,23 @@ "status": "provisional", "oracle": "none", "required_lanes": ["graph-parse", "tensor-ownership", "stage-load"] + }, + "workload-smoke": { + "status": "provisional", + "oracle": "none", + "required_lanes": ["class-specific-smoke"] + }, + "workload-oracle": { + "status": "certified", + "oracle": "local-monolithic", + "required_lanes": ["class-specific-smoke", "class-specific-oracle"] } } }, "models": [ { "family": "qwen3-dense", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "Qwen/Qwen3-0.6B-GGUF", "revision": "23749fefcc72300e3a2ad315e1317431b06b590a", "files": ["Qwen3-0.6B-Q8_0.gguf"], "file_integrity": {"Qwen3-0.6B-Q8_0.gguf": {"size_bytes": 639446688, "blob_id": "9465e63a22add5354d9bb4b99e90117043c7124007664907259bd16d043bb031"}}, "selector": "Q8_0"}, "execution": {"trunk_layers": 28, "mtp_layers": 0, "activation_width": 1024, "speculative_policy": "mtp-if-present"}, @@ -30,6 +41,7 @@ }, { "family": "llama", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "bartowski/Llama-3.2-1B-Instruct-GGUF", "revision": "067b946cf014b7c697f3654f621d577a3e3afd1c", "files": ["Llama-3.2-1B-Instruct-Q4_K_M.gguf"], "file_integrity": {"Llama-3.2-1B-Instruct-Q4_K_M.gguf": {"size_bytes": 807694464, "blob_id": "6f85a640a97cf2bf5b8e764087b1e83da0fdb51d7c9fab7d0fece9385611df83"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 16, "mtp_layers": 0, "activation_width": 2048, "speculative_policy": "mtp-if-present"}, @@ -38,6 +50,7 @@ }, { "family": "glm47-flash", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "unsloth/GLM-4.7-Flash-GGUF", "revision": "0d32489ecb9db6d2a4fc93bd27ef01519f95474d", "files": ["GLM-4.7-Flash-Q4_K_M.gguf"], "file_integrity": {"GLM-4.7-Flash-Q4_K_M.gguf": {"size_bytes": 18312339808, "blob_id": "29837ed2c0fc5f51981adf8ac8083fcf80743c598381f13e9f06cbad0498b174"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 47, "mtp_layers": 0, "activation_width": 2048, "speculative_policy": "mtp-if-present"}, @@ -46,6 +59,7 @@ }, { "family": "glm4", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "bartowski/glm-4-9b-chat-GGUF", "revision": "74a33eafa149550a60bff0fdafcdd34caa6df9f0", "files": ["glm-4-9b-chat-Q4_K_M.gguf"], "file_integrity": {"glm-4-9b-chat-Q4_K_M.gguf": {"size_bytes": 6250926848, "blob_id": "aa6cb8f5ef0a70399bdbf92eef566c26c6017ee0b131a424b65b5c757f1c81a2"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 40, "mtp_layers": 0, "activation_width": 4096, "speculative_policy": "mtp-if-present"}, @@ -54,6 +68,7 @@ }, { "family": "glm45-air", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "unsloth/GLM-4.5-Air-GGUF", "revision": "506d64aa8c5cfe9dbbf00bc7a15739438f83204d", "files": ["Q4_K_M/GLM-4.5-Air-Q4_K_M-00001-of-00002.gguf", "Q4_K_M/GLM-4.5-Air-Q4_K_M-00002-of-00002.gguf"], "file_integrity": {"Q4_K_M/GLM-4.5-Air-Q4_K_M-00001-of-00002.gguf": {"size_bytes": 50000746752, "blob_id": "3e7e5d25a6db33b7c90f8c5203d6f490d6c0a4f04a46f228af0e33727db5df5e"}, "Q4_K_M/GLM-4.5-Air-Q4_K_M-00002-of-00002.gguf": {"size_bytes": 22975001632, "blob_id": "ac6e50523d45c53faf3bf72d8fe109f3dd54bc6abe98a7773a3636112ead82a6"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 46, "mtp_layers": 1, "activation_width": 4096, "speculative_policy": "mtp-if-present"}, @@ -62,6 +77,7 @@ }, { "family": "deepseek2", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "bartowski/DeepSeek-Coder-V2-Lite-Instruct-GGUF", "revision": "8f248fa2072348f77a8bc37754e470de1f61866e", "files": ["DeepSeek-Coder-V2-Lite-Instruct-Q4_K_M.gguf"], "file_integrity": {"DeepSeek-Coder-V2-Lite-Instruct-Q4_K_M.gguf": {"size_bytes": 10364416768, "blob_id": "603bd3f8a0281d16571da7c08bd661ee17ff0d1be6fcbd1b42242da257ef0bb8"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 27, "mtp_layers": 0, "activation_width": 2048, "speculative_policy": "mtp-if-present"}, @@ -70,6 +86,7 @@ }, { "family": "gpt-oss", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "ggml-org/gpt-oss-20b-GGUF", "revision": "ef9b12f2ff56c69cf32153a02784e7a3c88bf524", "files": ["gpt-oss-20b-MXFP4.gguf"], "file_integrity": {"gpt-oss-20b-MXFP4.gguf": {"size_bytes": 12109566624, "blob_id": "27cd6c432c7672cb812a92f611cf3ba7bbc35928262bb1e1253ff4ee6ae35901"}}, "selector": "MXFP4"}, "execution": {"trunk_layers": 24, "mtp_layers": 0, "activation_width": 2880, "speculative_policy": "mtp-if-present"}, @@ -78,6 +95,7 @@ }, { "family": "qwen3-moe", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "unsloth/Qwen3-30B-A3B-Instruct-2507-GGUF", "revision": "eea7b2be5805a5f151f8847ede8e5f9a9284bf77", "files": ["Qwen3-30B-A3B-Instruct-2507-Q4_K_M.gguf"], "file_integrity": {"Qwen3-30B-A3B-Instruct-2507-Q4_K_M.gguf": {"size_bytes": 18556686752, "blob_id": "6c997b8af17debdfb01d890214400ccbab00db6acc0ba8da5de1cc906c4774d0"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 48, "mtp_layers": 0, "activation_width": 2048, "speculative_policy": "mtp-if-present"}, @@ -86,6 +104,7 @@ }, { "family": "qwen2-moe", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "RichardErkhov/Qwen_-_Qwen1.5-MoE-A2.7B-Chat-gguf", "revision": "947580e7b3904cd4081810d0efec4520a75609c7", "files": ["Qwen1.5-MoE-A2.7B-Chat.Q4_K_M.gguf"], "file_integrity": {"Qwen1.5-MoE-A2.7B-Chat.Q4_K_M.gguf": {"size_bytes": 9496236768, "blob_id": "3dcfb1032781788bf373edb49b888fcac5b6093ea0fe0edf5e23a977390b3eea"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 24, "mtp_layers": 0, "activation_width": 2048, "speculative_policy": "mtp-if-present"}, @@ -94,6 +113,7 @@ }, { "family": "falcon-h1", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "tiiuae/Falcon-H1-1.5B-Instruct-GGUF", "revision": "0d3a6cfe25fb4eeab0153fb8623aac5b69d6bd0a", "files": ["Falcon-H1-1.5B-Instruct-Q4_K_M.gguf"], "file_integrity": {"Falcon-H1-1.5B-Instruct-Q4_K_M.gguf": {"size_bytes": 944786656, "blob_id": "8b51aa2aa34a0373fd0cd64c02eb91d1bc1da681c09e955ad769d4a9b2d8385f"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 24, "mtp_layers": 0, "activation_width": 2048, "speculative_policy": "mtp-if-present"}, @@ -102,6 +122,7 @@ }, { "family": "jamba2", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "bartowski/ai21labs_AI21-Jamba2-3B-GGUF", "revision": "02d70acd708332ec4e78e9ceefe116851a307411", "files": ["ai21labs_AI21-Jamba2-3B-Q4_K_M.gguf"], "file_integrity": {"ai21labs_AI21-Jamba2-3B-Q4_K_M.gguf": {"size_bytes": 1932696448, "blob_id": "fa0876fa152f38689cefd6d498907d6255cf515780d65b16346d3190df6a0794"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 28, "mtp_layers": 0, "activation_width": 2560, "speculative_policy": "mtp-if-present"}, @@ -110,6 +131,7 @@ }, { "family": "kimi-linear", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "bartowski/moonshotai_Kimi-Linear-48B-A3B-Instruct-GGUF", "revision": "228dbe476e5a02091624a19068f4c962caa8a1c5", "files": ["moonshotai_Kimi-Linear-48B-A3B-Instruct-IQ2_XS.gguf"], "file_integrity": {"moonshotai_Kimi-Linear-48B-A3B-Instruct-IQ2_XS.gguf": {"size_bytes": 13897237408, "blob_id": "883be5561005ed6558c3f9d58d33bd9df818ea41c18086a49769047278953c0e"}}, "selector": "IQ2_XS"}, "execution": {"trunk_layers": 27, "mtp_layers": 0, "activation_width": 2304, "speculative_policy": "mtp-if-present"}, @@ -118,6 +140,7 @@ }, { "family": "qwen3-next", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "bartowski/Qwen_Qwen3-Next-80B-A3B-Thinking-GGUF", "revision": "c36ac5f15be18742fcd09baa61d48dfc016aae79", "files": ["Qwen_Qwen3-Next-80B-A3B-Thinking-IQ2_XS.gguf"], "file_integrity": {"Qwen_Qwen3-Next-80B-A3B-Thinking-IQ2_XS.gguf": {"size_bytes": 22217422720, "blob_id": "ece0ffe31891d1a0f1993565db4a977ae6d862d0ae2e45fc7282ff0a06cf5d5d"}}, "selector": "IQ2_XS"}, "execution": {"trunk_layers": 48, "mtp_layers": 0, "activation_width": 2048, "speculative_policy": "mtp-if-present"}, @@ -126,6 +149,7 @@ }, { "family": "qwen4exp", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "unsloth/Qwen3.8-Flash-Next-GGUF", "revision": "c8b5954a88c2775c546b92593eda40ea041d3176", "files": ["UD-IQ1_S/Qwen3.8-Flash-Next-UD-IQ1_S-00001-of-00003.gguf", "UD-IQ1_S/Qwen3.8-Flash-Next-UD-IQ1_S-00002-of-00003.gguf", "UD-IQ1_S/Qwen3.8-Flash-Next-UD-IQ1_S-00003-of-00003.gguf"], "file_integrity": {"UD-IQ1_S/Qwen3.8-Flash-Next-UD-IQ1_S-00001-of-00003.gguf": {"size_bytes": 10946624, "blob_id": "88a1420825a9304063e882ada29d438263617f51ac8923d438d927496693bafd"}, "UD-IQ1_S/Qwen3.8-Flash-Next-UD-IQ1_S-00002-of-00003.gguf": {"size_bytes": 49990818368, "blob_id": "3a62e35bbf9add4733bd1438ebd3a67649d5edd6cb0e72bb78e33c913992b2b6"}, "UD-IQ1_S/Qwen3.8-Flash-Next-UD-IQ1_S-00003-of-00003.gguf": {"size_bytes": 22544696352, "blob_id": "0e25ceaeb89b8a80aa973c6c0c7448943682f7408c2855b2ebd016b7643a861a"}}, "selector": "UD-IQ1_S"}, "execution": {"trunk_layers": 48, "mtp_layers": 0, "activation_width": 10240, "speculative_policy": "mtp-if-present"}, @@ -134,6 +158,7 @@ }, { "family": "nemotron", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF", "revision": "f2d3fe3694501008786e81e5f20360cbf715496a", "files": ["NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q4_K_M.gguf"], "file_integrity": {"NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q4_K_M.gguf": {"size_bytes": 25266255936, "blob_id": "edcb5d4650796ed2fb412498de6f83b585862312c747ddb74f0ea04b22206181"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 52, "mtp_layers": 1, "activation_width": 2688, "speculative_policy": "mtp-if-present"}, @@ -142,6 +167,7 @@ }, { "family": "mamba", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "Felladrin/gguf-mamba-130m-hf", "revision": "83ef2222e1a0437d26bda213537afc195a53b3ad", "files": ["mamba-130m-hf.Q4_K_M.gguf"], "file_integrity": {"mamba-130m-hf.Q4_K_M.gguf": {"size_bytes": 103576544, "blob_id": "2784131bc0546f802bcb720cbc77d26c174c85514400cc645507146ad95f07da"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 24, "mtp_layers": 0, "activation_width": 768, "speculative_policy": "mtp-if-present"}, @@ -150,6 +176,7 @@ }, { "family": "mamba2", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "bartowski/falcon-mamba-7b-GGUF", "revision": "1236a076ebddbcfcbd6ba0fe310d473d20527f11", "files": ["falcon-mamba-7b-Q4_K_M.gguf"], "file_integrity": {"falcon-mamba-7b-Q4_K_M.gguf": {"size_bytes": 4204230656, "blob_id": "ef8ba78a1fd0b0fd1d1c966acc5f4c043b12fbbd7044f4d9aad82e6b7a51e0a7"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 64, "mtp_layers": 0, "activation_width": 4096, "speculative_policy": "mtp-if-present"}, @@ -158,6 +185,7 @@ }, { "family": "rwkv6", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "latestissue/rwkv-6-finch-1b6-gguf", "revision": "9f4420bafb8fe463599b9983d81e12e1868c3746", "files": ["rwkv-6-finch-1b6-Q4_0.gguf"], "file_integrity": {"rwkv-6-finch-1b6-Q4_0.gguf": {"size_bytes": 992738848, "blob_id": "7b345728c3792f0168a615bd529a319d63f8becb760090330ce583a24f55e46b"}}, "selector": "Q4_0"}, "execution": {"trunk_layers": 24, "mtp_layers": 0, "activation_width": 2048, "speculative_policy": "mtp-if-present"}, @@ -166,6 +194,7 @@ }, { "family": "rwkv7", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "Mungert/RWKV7-Goose-World3-2.9B-HF-GGUF", "revision": "e85c69858772a4d6b481d99e1e49910dad48e123", "files": ["RWKV7-Goose-World3-2.9B-HF-f16-q4_k.gguf"], "file_integrity": {"RWKV7-Goose-World3-2.9B-HF-f16-q4_k.gguf": {"size_bytes": 2314884576, "blob_id": "bfcb5b239cf99ac4a590d7163266bbe3bb14f41d221ad6f7a2bf602b610211b0"}}, "selector": "Q4_K"}, "execution": {"trunk_layers": 32, "mtp_layers": 0, "activation_width": 2560, "speculative_policy": "mtp-if-present"}, @@ -174,6 +203,7 @@ }, { "family": "granite-hybrid", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "ibm-granite/granite-4.0-h-350m-GGUF", "revision": "a864f823cce6e6048b5752e2816fe7a23987d790", "files": ["granite-4.0-h-350m-Q4_K_M.gguf"], "file_integrity": {"granite-4.0-h-350m-Q4_K_M.gguf": {"size_bytes": 222662560, "blob_id": "0a8d6a7373602fadfba274a640ba784b86cc6847f1c67f1b0a90fa2ec266b7fb"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 32, "mtp_layers": 0, "activation_width": 768, "speculative_policy": "mtp-if-present"}, @@ -182,6 +212,7 @@ }, { "family": "qwen35", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "unsloth/Qwen3.5-0.8B-GGUF", "revision": "6ab461498e2023f6e3c1baea90a8f0fe38ab64d0", "files": ["Qwen3.5-0.8B-Q4_K_M.gguf"], "file_integrity": {"Qwen3.5-0.8B-Q4_K_M.gguf": {"size_bytes": 532517120, "blob_id": "bd258782e35f7f458f8aced1adc053e6e92e89bc735ba3be89d38a06121dc517"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 24, "mtp_layers": 0, "activation_width": 1024, "speculative_policy": "mtp-if-present"}, @@ -190,6 +221,7 @@ }, { "family": "granite-dense", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "bartowski/ibm-granite_granite-3.2-2b-instruct-GGUF", "revision": "9be2c106c8c073f5a140ec03c8c7a6a6e72d097b", "files": ["ibm-granite_granite-3.2-2b-instruct-Q4_K_M.gguf"], "file_integrity": {"ibm-granite_granite-3.2-2b-instruct-Q4_K_M.gguf": {"size_bytes": 1545296512, "blob_id": "e1b915b0849becf4fdda188dee7b09cbebbfabd71c6f3f2b75dd3eca0a8fded1"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 40, "mtp_layers": 0, "activation_width": 2048, "speculative_policy": "mtp-if-present"}, @@ -198,6 +230,7 @@ }, { "family": "lfm2", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "LiquidAI/LFM2-350M-GGUF", "revision": "8fdc9d526b7ed346b19257551b05816c7912ecc2", "files": ["LFM2-350M-Q4_K_M.gguf"], "file_integrity": {"LFM2-350M-Q4_K_M.gguf": {"size_bytes": 229309376, "blob_id": "a4d000c7064bd3b2e42c6845836286a899a4e79cf1791da1a6797b58d575957d"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 16, "mtp_layers": 0, "activation_width": 1024, "speculative_policy": "mtp-if-present"}, @@ -206,6 +239,7 @@ }, { "family": "lfm2-vl", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "ggml-org/LFM2-VL-450M-GGUF", "revision": "0ff2ce5a6710f9bf0a6de87ec91832b43fa0ac05", "files": ["LFM2-VL-450M-Q8_0.gguf"], "file_integrity": {"LFM2-VL-450M-Q8_0.gguf": {"size_bytes": 379215264, "blob_id": "e97704a0cf0a1d00ca604b4c672c82f4234318dba9d43a7f7a4c0d2df6747844"}}, "selector": "Q8_0"}, "mmproj_artifact": {"repo": "ggml-org/LFM2-VL-450M-GGUF", "revision": "0ff2ce5a6710f9bf0a6de87ec91832b43fa0ac05", "files": ["mmproj-LFM2-VL-450M-Q8_0.gguf"], "file_integrity": {"mmproj-LFM2-VL-450M-Q8_0.gguf": {"size_bytes": 103890016, "blob_id": "a6a813fae4ba3f90852a990fdd4786a017dba0e80e599a366853ba539ee2b5ef"}}, "selector": "Q8_0"}, @@ -215,6 +249,7 @@ }, { "family": "laguna", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "poolside/Laguna-XS-2.1-GGUF", "revision": "1a37c0a5fb8c7a18e6106decb6be6327d1b63fa6", "files": ["Laguna-XS-2.1-Q4_K_M.gguf"], "file_integrity": {"Laguna-XS-2.1-Q4_K_M.gguf": {"size_bytes": 20274300032, "blob_id": "1ac7079101fca5a6df8c5a7523a3c30ea7d1c0e4b1258090e7d6d4039287f6cb"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 40, "mtp_layers": 0, "activation_width": 2048, "speculative_policy": "mtp-if-present"}, @@ -223,6 +258,7 @@ }, { "family": "qwen2-vl", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "bartowski/Qwen2-VL-2B-Instruct-GGUF", "revision": "2160e265f926e84e84dbb1c73e623b5324b23707", "files": ["Qwen2-VL-2B-Instruct-Q4_K_M.gguf"], "file_integrity": {"Qwen2-VL-2B-Instruct-Q4_K_M.gguf": {"size_bytes": 986047232, "blob_id": "4ef095263343fc1237e8ca879790bb262bcf209f082e0a9bfce219b7ece55e8b"}}, "selector": "Q4_K_M"}, "mmproj_artifact": {"repo": "bartowski/Qwen2-VL-2B-Instruct-GGUF", "revision": "2160e265f926e84e84dbb1c73e623b5324b23707", "files": ["mmproj-Qwen2-VL-2B-Instruct-f16.gguf"], "file_integrity": {"mmproj-Qwen2-VL-2B-Instruct-f16.gguf": {"size_bytes": 1331656192, "blob_id": "05cc3ae461a7b6aa4023312ccab549ecab77cf8677efee04f049fcbab55b8bc3"}}, "selector": "f16"}, @@ -232,6 +268,7 @@ }, { "family": "qwen3-vl", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "unsloth/Qwen3-VL-8B-Instruct-GGUF", "revision": "b93a7ee713758252c555be4210c00540df954dc2", "files": ["Qwen3-VL-8B-Instruct-Q4_K_M.gguf"], "file_integrity": {"Qwen3-VL-8B-Instruct-Q4_K_M.gguf": {"size_bytes": 5027785568, "blob_id": "108e7ff92b78eefd3db4741885104acba514255c11b617d3c7b197a5f46efe89"}}, "selector": "Q4_K_M"}, "mmproj_artifact": {"repo": "unsloth/Qwen3-VL-8B-Instruct-GGUF", "revision": "b93a7ee713758252c555be4210c00540df954dc2", "files": ["mmproj-F16.gguf"], "file_integrity": {"mmproj-F16.gguf": {"size_bytes": 1159030336, "blob_id": "d406d03ebabefdef86a2c86bf0c1b65f9e046f7a81c218f25de4931b46a07fc4"}}, "selector": "F16"}, @@ -241,6 +278,7 @@ }, { "family": "gemma2", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "google/gemma-2-2b-it-GGUF", "revision": "67803c710cb0c89bae2047a5f50a2481c9339d24", "files": ["2b_it_v2.gguf"], "file_integrity": {"2b_it_v2.gguf": {"size_bytes": 10463413632, "blob_id": "8fc1a799613b873edd085ffcf33fd710077daf03833ead29e136cca9b528ecb6"}}, "selector": "2b_it_v2"}, "execution": {"trunk_layers": 26, "mtp_layers": 0, "activation_width": 2304, "speculative_policy": "mtp-if-present"}, @@ -249,6 +287,7 @@ }, { "family": "phi3", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "microsoft/Phi-3-mini-4k-instruct-gguf", "revision": "a64113399c2f6b8ad3e11c394733a2ddadaa7f33", "files": ["Phi-3-mini-4k-instruct-q4.gguf"], "file_integrity": {"Phi-3-mini-4k-instruct-q4.gguf": {"size_bytes": 2393231072, "blob_id": "8a83c7fb9049a9b2e92266fa7ad04933bb53aa1e85136b7b30f1b8000ff2edef"}}, "selector": "q4"}, "execution": {"trunk_layers": 32, "mtp_layers": 0, "activation_width": 3072, "speculative_policy": "mtp-if-present"}, @@ -257,6 +296,7 @@ }, { "family": "phi4", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "unsloth/phi-4-GGUF", "revision": "5110b7771e8166d5530e73346a15aea096a8cb99", "files": ["phi-4-Q4_K_M.gguf"], "file_integrity": {"phi-4-Q4_K_M.gguf": {"size_bytes": 8890306112, "blob_id": "01e1f25b3e6931054c6c2227b06f4969828434eebc299e8e171f55dab6814485"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 40, "mtp_layers": 0, "activation_width": 5120, "speculative_policy": "mtp-if-present"}, @@ -265,6 +305,7 @@ }, { "family": "falcon-dense", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "maddes8cht/tiiuae-falcon-7b-instruct-gguf", "revision": "56d71fbb3ea040a1d35459013bd2688613075668", "files": ["tiiuae-falcon-7b-instruct-Q4_K_M.gguf"], "file_integrity": {"tiiuae-falcon-7b-instruct-Q4_K_M.gguf": {"size_bytes": 4975385792, "blob_id": "6f6c886ed07d4f6133a4dd7f8b799764ed719c50973d96816b2ac5ff3ada9913"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 32, "mtp_layers": 0, "activation_width": 4544, "speculative_policy": "mtp-if-present"}, @@ -273,6 +314,7 @@ }, { "family": "internlm2", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "Felladrin/gguf-internlm2-chat-1_8b", "revision": "cdd579415ed57fca8af5c5e09b35ee5124449876", "files": ["internlm2-chat-1_8b.F16.gguf"], "file_integrity": {"internlm2-chat-1_8b.F16.gguf": {"size_bytes": 3780559520, "blob_id": "377e44d5967ea2a51570fbc28448dd8693d0139c73b13b9e773fad783f7bd957"}}, "selector": "F16"}, "execution": {"trunk_layers": 24, "mtp_layers": 0, "activation_width": 2048, "speculative_policy": "mtp-if-present"}, @@ -281,6 +323,7 @@ }, { "family": "mistral-small", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "unsloth/Mistral-Small-3.2-24B-Instruct-2506-GGUF", "revision": "b750ec2299225e492f1bd27cab88a0a595fa848f", "files": ["Mistral-Small-3.2-24B-Instruct-2506-Q4_K_M.gguf"], "file_integrity": {"Mistral-Small-3.2-24B-Instruct-2506-Q4_K_M.gguf": {"size_bytes": 14333922848, "blob_id": "a3cc56310807ed0d145eaf9f018ccda9ae7ad8edb41ec870aa2454b0d4700b3c"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 40, "mtp_layers": 0, "activation_width": 5120, "speculative_policy": "mtp-if-present"}, @@ -289,6 +332,7 @@ }, { "family": "cohere2", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "bartowski/c4ai-command-r7b-12-2024-GGUF", "revision": "bfc7a934c45cb839d84c8ca01d87f1cfa51aaa3f", "files": ["c4ai-command-r7b-12-2024-Q4_K_M.gguf"], "file_integrity": {"c4ai-command-r7b-12-2024-Q4_K_M.gguf": {"size_bytes": 5057009792, "blob_id": "0653c1915f7e4819a980deea8e9ba86b0a78cbb283712785cbf091c3ffbe3b13"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 32, "mtp_layers": 0, "activation_width": 4096, "speculative_policy": "mtp-if-present"}, @@ -297,6 +341,7 @@ }, { "family": "smollm3", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "ggml-org/SmolLM3-3B-GGUF", "revision": "4965cb60b150737b68a0408c36aeefb65078f894", "files": ["SmolLM3-Q4_K_M.gguf"], "file_integrity": {"SmolLM3-Q4_K_M.gguf": {"size_bytes": 1915305312, "blob_id": "8334b850b7bd46238c16b0c550df2138f0889bf433809008cc17a8b05761863e"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 36, "mtp_layers": 0, "activation_width": 2048, "speculative_policy": "mtp-if-present"}, @@ -305,6 +350,7 @@ }, { "family": "qwen2", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "Qwen/Qwen2-0.5B-Instruct-GGUF", "revision": "198f08841147e5196a6a69bd0053690fb1fd3857", "files": ["qwen2-0_5b-instruct-q4_k_m.gguf"], "file_integrity": {"qwen2-0_5b-instruct-q4_k_m.gguf": {"size_bytes": 397805248, "blob_id": "f0a42bb979ca62b5e61f3bf924ab4b6a40aa091825ee7dcb4039949980ab81a8"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 24, "mtp_layers": 0, "activation_width": 896, "speculative_policy": "disabled"}, @@ -313,6 +359,7 @@ }, { "family": "bloom", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "afrideva/bloom-560m-GGUF", "revision": "fe8ad7e97957783ea24f6960f73e28e709aa6e96", "files": ["bloom-560m.q4_k_m.gguf"], "file_integrity": {"bloom-560m.q4_k_m.gguf": {"size_bytes": 561444928, "blob_id": "383bc73de0c2e96704301b27117e7fd209255eda0474198dafdf4dd61ad7bf3e"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 24, "mtp_layers": 0, "activation_width": 1024, "speculative_policy": "disabled"}, @@ -321,6 +368,7 @@ }, { "family": "gemma", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "lmstudio-ai/gemma-2b-it-GGUF", "revision": "a0b140bfb922a743f89dd0682a24a17516071ab9", "files": ["gemma-2b-it-q4_k_m.gguf"], "file_integrity": {"gemma-2b-it-q4_k_m.gguf": {"size_bytes": 1495245728, "blob_id": "144b3eb4fb500556034a29172cfa2b17f28cfce7c7698ac3467600adc510efcd"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 18, "mtp_layers": 0, "activation_width": 2048, "speculative_policy": "disabled"}, @@ -329,6 +377,7 @@ }, { "family": "gemma3", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "ggml-org/gemma-3-1b-it-GGUF", "revision": "f9c28bcd85737ffc5aef028638d3341d49869c27", "files": ["gemma-3-1b-it-Q4_K_M.gguf"], "file_integrity": {"gemma-3-1b-it-Q4_K_M.gguf": {"size_bytes": 806058240, "blob_id": "8ccc5cd1f1b3602548715ae25a66ed73fd5dc68a210412eea643eb20eb75a135"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 26, "mtp_layers": 0, "activation_width": 1152, "speculative_policy": "disabled"}, @@ -337,6 +386,7 @@ }, { "family": "phi2", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "TheBloke/phi-2-GGUF", "revision": "5a454d977c6438bb9fb2df233c8ca70f21c87420", "files": ["phi-2.Q4_K_M.gguf"], "file_integrity": {"phi-2.Q4_K_M.gguf": {"size_bytes": 1789239136, "blob_id": "324356668fa5ba9f4135de348447bb2bbe2467eaa1b8fcfb53719de62fbd2499"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 32, "mtp_layers": 0, "activation_width": 2560, "speculative_policy": "disabled"}, @@ -345,6 +395,7 @@ }, { "family": "starcoder2", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "second-state/StarCoder2-3B-GGUF", "revision": "7fca3e2da2ce31df411461e2cb9cae2d2b492f35", "files": ["starcoder2-3b-Q4_K_M.gguf"], "file_integrity": {"starcoder2-3b-Q4_K_M.gguf": {"size_bytes": 1848976448, "blob_id": "d8fb39287a463549b80d97473b0a7595c3a5a6da3ae2604ca33906a1a43f7175"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 30, "mtp_layers": 0, "activation_width": 3072, "speculative_policy": "disabled"}, @@ -353,6 +404,7 @@ }, { "family": "stablelm", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "second-state/stablelm-2-zephyr-1.6b-GGUF", "revision": "6da77f49f48f36f1a01753ca27ba531bde13b861", "files": ["stablelm-2-zephyr-1_6b-Q4_K_M.gguf"], "file_integrity": {"stablelm-2-zephyr-1_6b-Q4_K_M.gguf": {"size_bytes": 1031442432, "blob_id": "4224a8a08417e34d5a224efb1291491c4f5762b46aa012021adaf0a0578848e2"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 24, "mtp_layers": 0, "activation_width": 2048, "speculative_policy": "disabled"}, @@ -361,6 +413,7 @@ }, { "family": "olmo2", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "allenai/OLMo-2-0425-1B-Instruct-GGUF", "revision": "62f8c199538474c3e33ed5d7e0580abd66686a27", "files": ["OLMo-2-0425-1B-Instruct-Q4_K_M.gguf"], "file_integrity": {"OLMo-2-0425-1B-Instruct-Q4_K_M.gguf": {"size_bytes": 935515296, "blob_id": "abd8187934a438fbf7cfff0a1de5b9d2793ce913f158794df1951dcba6c93cc6"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 16, "mtp_layers": 0, "activation_width": 2048, "speculative_policy": "disabled"}, @@ -369,6 +422,7 @@ }, { "family": "exaone4", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "LGAI-EXAONE/EXAONE-4.0-1.2B-GGUF", "revision": "162446400ea4596377a3ce1d3ddffa32971af0a6", "files": ["EXAONE-4.0-1.2B-Q4_K_M.gguf"], "file_integrity": {"EXAONE-4.0-1.2B-Q4_K_M.gguf": {"size_bytes": 812437792, "blob_id": "7b5e753540183ae4d56e6febd9b48cdd944de53386e6faa8f51c8f98cb2b47df"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 30, "mtp_layers": 0, "activation_width": 2048, "speculative_policy": "disabled"}, @@ -377,6 +431,7 @@ }, { "family": "minicpm3", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "openbmb/MiniCPM3-4B-GGUF", "revision": "816dc79b35f92827e0d2d87aacea3567e49661a8", "files": ["minicpm3-4b-q4_k_m.gguf"], "file_integrity": {"minicpm3-4b-q4_k_m.gguf": {"size_bytes": 2469791584, "blob_id": "64913247e927414ecf47fd3e9ea8e3f0c9acae293f583dfa7e24b8872e20fa4c"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 62, "mtp_layers": 0, "activation_width": 2560, "speculative_policy": "disabled"}, @@ -385,6 +440,7 @@ }, { "family": "nemotron-expansion", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "bartowski/Nemotron-Mini-4B-Instruct-GGUF", "revision": "fb49cde090c86092d89905bea2ffc41c23c2615e", "files": ["Nemotron-Mini-4B-Instruct-Q4_K_M.gguf"], "file_integrity": {"Nemotron-Mini-4B-Instruct-Q4_K_M.gguf": {"size_bytes": 2697387072, "blob_id": "2bf02846dbd45e9580b9338b90505fb54e00d185713d4b7f06699e2d5d298e48"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 32, "mtp_layers": 0, "activation_width": 3072, "speculative_policy": "disabled"}, @@ -393,6 +449,7 @@ }, { "family": "arcee", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "arcee-ai/AFM-4.5B-GGUF", "revision": "741fded55739927f94be7934a174c2bcfc751969", "files": ["AFM-4.5B-Q4_K_M.gguf"], "file_integrity": {"AFM-4.5B-Q4_K_M.gguf": {"size_bytes": 2916301824, "blob_id": "f05516b323f581bebae1af2cbf900d83a2569b0a60c54366daf4a9c15ae30d4f"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 36, "mtp_layers": 0, "activation_width": 2560, "speculative_policy": "disabled"}, @@ -401,6 +458,7 @@ }, { "family": "mpt", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "maddes8cht/mosaicml-mpt-7b-instruct-gguf", "revision": "939bf1d3b1fcab272e542599bfaf74dcd04a9931", "files": ["mosaicml-mpt-7b-instruct-Q4_K_M.gguf"], "file_integrity": {"mosaicml-mpt-7b-instruct-Q4_K_M.gguf": {"size_bytes": 4390524128, "blob_id": "bfc1cf1333d60dd8f4c9ea97652ebdb984501c389d22cebc02b564f45c68d628"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 32, "mtp_layers": 0, "activation_width": 4096, "speculative_policy": "disabled"}, @@ -409,6 +467,7 @@ }, { "family": "apertus", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "bartowski/swiss-ai_Apertus-8B-Instruct-2509-GGUF", "revision": "d2ffbcc459d0a1bccb2097aca8bb825974f7678c", "files": ["swiss-ai_Apertus-8B-Instruct-2509-Q4_K_M.gguf"], "file_integrity": {"swiss-ai_Apertus-8B-Instruct-2509-Q4_K_M.gguf": {"size_bytes": 5057885568, "blob_id": "5901007f15aec9aabeea401cbe65559671166fc851bbdf8c26f56c2b61e9cd1d"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 32, "mtp_layers": 0, "activation_width": 4096, "speculative_policy": "disabled"}, @@ -417,6 +476,7 @@ }, { "family": "glm4-expansion", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "unsloth/GLM-4-9B-0414-GGUF", "revision": "40a17a0c8f24664ddd851dafd03935553c25a57e", "files": ["GLM-4-9B-0414-Q4_K_M.gguf"], "file_integrity": {"GLM-4-9B-0414-Q4_K_M.gguf": {"size_bytes": 6166574944, "blob_id": "8027e1089273e8817b2df0d91c9aa17c5ea467246dcdacac34989f8919fe6540"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 40, "mtp_layers": 0, "activation_width": 4096, "speculative_policy": "disabled"}, @@ -425,6 +485,7 @@ }, { "family": "afmoe", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "MaziyarPanahi/Trinity-Mini-GGUF", "revision": "3a7d7a64abeabff7302851a2bc98508b36d0b8f7", "files": ["Trinity-Mini.Q4_K_M.gguf"], "file_integrity": {"Trinity-Mini.Q4_K_M.gguf": {"size_bytes": 15823053440, "blob_id": "2c43474de9017d68331455cbf3852c1d365f7a5f9d3b050418e024ba3374a19e"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 32, "mtp_layers": 0, "activation_width": 2048, "speculative_policy": "mtp-if-present"}, @@ -433,6 +494,7 @@ }, { "family": "arwkv7", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "mradermacher/ARWKV-7B-Preview-0.1-NoG-32B-i1-GGUF", "revision": "301a9554c8271bca175a78f599b20e49bce383fa", "files": ["ARWKV-7B-Preview-0.1-NoG-32B.i1-Q4_K_M.gguf"], "file_integrity": {"ARWKV-7B-Preview-0.1-NoG-32B.i1-Q4_K_M.gguf": {"size_bytes": 5117561504, "blob_id": "841c66e3242e2a7e1b64641e1ff92248b2650129b196dc3e27c016f541cb0059"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 28, "mtp_layers": 0, "activation_width": 3584, "speculative_policy": "mtp-if-present"}, @@ -441,6 +503,7 @@ }, { "family": "bailingmoe3", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "SC117/Ling-3.0-tiny-abliterated-APEX-GGUF", "revision": "b923d16fcf28261f12be9ece2b520ed442403f70", "files": ["Ling-3.0-tiny-abliterated-bf16.gguf"], "file_integrity": {"Ling-3.0-tiny-abliterated-bf16.gguf": {"size_bytes": 15803475200, "blob_id": "8c2ea2ead488a0e00bcc02c41fd144d6226ef70eb626afcd589e3960ce39fc85"}}, "selector": "BF16"}, "execution": {"trunk_layers": 24, "mtp_layers": 0, "activation_width": 1536, "speculative_policy": "mtp-if-present"}, @@ -449,6 +512,7 @@ }, { "family": "chameleon", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "mradermacher/AA-chameleon-7b-plus-i1-GGUF", "revision": "5e13750981e775965061feefb701f4b6e1e1f7d9", "files": ["AA-chameleon-7b-plus.i1-Q4_K_M.gguf"], "file_integrity": {"AA-chameleon-7b-plus.i1-Q4_K_M.gguf": {"size_bytes": 4274466400, "blob_id": "6af766cf8ac2c729c06c430eabc506e32ce10d237c92b426742b3b4cceabcd37"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 32, "mtp_layers": 0, "activation_width": 4096, "speculative_policy": "mtp-if-present"}, @@ -457,6 +521,7 @@ }, { "family": "deepseek32", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "sszymczyk/DeepSeek-V3.2-4Layers-GGUF", "revision": "54c8e50a9bfaced560a51eeb63cace952e3ea247", "files": ["DeepSeek-V3.2-4Layers-Q8_0.gguf"], "file_integrity": {"DeepSeek-V3.2-4Layers-Q8_0.gguf": {"size_bytes": 16131110560, "blob_id": "a7587dc5291aab8ce6d806c1046a169c499fb39008d4188acbca223fd6ad686d"}}, "selector": "Q8_0"}, "execution": {"trunk_layers": 4, "mtp_layers": 0, "activation_width": 7168, "speculative_policy": "mtp-if-present"}, @@ -465,6 +530,7 @@ }, { "family": "dots1", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "Robertp423/dots.llm1.test-Q6_K-GGUF", "revision": "a0404996541e6505cad00b7e3bfa4c7447f4f501", "files": ["dots.llm1.test-q6_k.gguf"], "file_integrity": {"dots.llm1.test-q6_k.gguf": {"size_bytes": 14302555328, "blob_id": "2789482e96ffe36999a3c32c1224f0747c030f7aefe576a3a8510a0aa84fde7e"}}, "selector": "Q6_K"}, "execution": {"trunk_layers": 27, "mtp_layers": 0, "activation_width": 2048, "speculative_policy": "mtp-if-present"}, @@ -473,6 +539,7 @@ }, { "family": "ernie4-5", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "Arm/ernie-4-5-0-3b-pt-q4-k-m-llamacpp-raspberrypi5", "revision": "86a80f7bb8780cdc73bed0ffbaab3f6ed98c8b63", "files": ["baidu__ERNIE-4.5-0.3B-PT_llamacpp_optimized.gguf"], "file_integrity": {"baidu__ERNIE-4.5-0.3B-PT_llamacpp_optimized.gguf": {"size_bytes": 240645504, "blob_id": "86202506dbbffaddaaa615c932676d3e4c21d1f93e11cd1da3ffb9c6d10cbaed"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 18, "mtp_layers": 0, "activation_width": 1024, "speculative_policy": "mtp-if-present"}, @@ -481,6 +548,7 @@ }, { "family": "ernie4-5-moe", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "mradermacher/ERNIE-21B-A3B-Claude-4.5-High-OPUS-Thinking-i1-GGUF", "revision": "51b8fa4872193aee98ddf4ae60c63aef52e2a07a", "files": ["ERNIE-21B-A3B-Claude-4.5-High-OPUS-Thinking.i1-Q4_K_M.gguf"], "file_integrity": {"ERNIE-21B-A3B-Claude-4.5-High-OPUS-Thinking.i1-Q4_K_M.gguf": {"size_bytes": 13245833216, "blob_id": "5658d5e0b82578ea7ec0a40d022cbed31013dc6bb0da11c11d16261915b7bcf3"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 28, "mtp_layers": 0, "activation_width": 2560, "speculative_policy": "mtp-if-present"}, @@ -489,6 +557,7 @@ }, { "family": "gemma3n", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "lmstudio-community/gemma-3n-E4B-it-text-GGUF", "revision": "48ddb4928f7910916a599054277dac92ab70201a", "files": ["gemma-3n-E4B-it-Q4_K_M.gguf"], "file_integrity": {"gemma-3n-E4B-it-Q4_K_M.gguf": {"size_bytes": 4237063328, "blob_id": "7fcb647151fa19a0750538672cf824ef6cf18f74bb86ebe5592e1ed59b4070a0"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 35, "mtp_layers": 0, "activation_width": 2048, "speculative_policy": "mtp-if-present"}, @@ -497,6 +566,7 @@ }, { "family": "gemma4", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "HauhauCS/Gemma-4-E4B-Uncensored-HauhauCS-Aggressive", "revision": "45b6a334b4bcd1d7f37179df58b3b1d66a184e5d", "files": ["Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf"], "file_integrity": {"Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf": {"size_bytes": 5335285728, "blob_id": "d0027dd3a9128d9323e9f282c8bf010a8526c46477584535991dc1a869b56e96"}}, "selector": "Q4_K_M"}, "mmproj_artifact": {"repo": "HauhauCS/Gemma-4-E4B-Uncensored-HauhauCS-Aggressive", "revision": "45b6a334b4bcd1d7f37179df58b3b1d66a184e5d", "files": ["mmproj-Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-f16.gguf"], "file_integrity": {"mmproj-Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-f16.gguf": {"size_bytes": 990288832, "blob_id": "debad39ab9c1152ab67695a674fb35e8375b2320c57bfd5075835d3ccb16c7db"}}, "selector": "F16"}, @@ -506,6 +576,7 @@ }, { "family": "gptneox", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "mradermacher/gpt-neox-20b-i1-GGUF", "revision": "aea1d77733d7cef1ccc4556c63666d17e5f95a28", "files": ["gpt-neox-20b.i1-Q4_K_M.gguf"], "file_integrity": {"gpt-neox-20b.i1-Q4_K_M.gguf": {"size_bytes": 13135032320, "blob_id": "1a35c13535c26bb925a2379d9f021de67a0390124ffe2051aa94412002652d6d"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 44, "mtp_layers": 0, "activation_width": 6144, "speculative_policy": "mtp-if-present"}, @@ -514,6 +585,7 @@ }, { "family": "granite-swa", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "emredeveloper/granite-swash-3b-a600m-GGUF", "revision": "d20c88f11ef8622464f9678e6437ed9e72a07dcd", "files": ["granite-swash-3b-a600m-BF16.gguf"], "file_integrity": {"granite-swash-3b-a600m-BF16.gguf": {"size_bytes": 6047918368, "blob_id": "1961c3d8eb4e97536a2b95ebe0bd6830d508031443e81124516c05505af8c615"}}, "selector": "BF16"}, "execution": {"trunk_layers": 28, "mtp_layers": 0, "activation_width": 1280, "speculative_policy": "mtp-if-present"}, @@ -522,6 +594,7 @@ }, { "family": "granite-moe", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "sinatras/granite-moe-1b-split", "revision": "3beccd31cf6d51f94c93007e56a21e0df41a101a", "files": ["Q4_K_M/granite-3.1-1b-a400m-instruct-Q4_K_M.gguf"], "file_integrity": {"Q4_K_M/granite-3.1-1b-a400m-instruct-Q4_K_M.gguf": {"size_bytes": 821847360, "blob_id": "3a2ec1c2a78cb29d901e29bbf5162dcd03381e13803d2cbdcff838d4d08142eb"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 24, "mtp_layers": 0, "activation_width": 1024, "speculative_policy": "mtp-if-present"}, @@ -530,6 +603,7 @@ }, { "family": "granite-switch", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "barha/granite-switch-4.1-3b-preview-GGUF", "revision": "1537da1c12cd890ef0e632678221ecd663485d4e", "files": ["granite-switch-4.1-3b-preview-f16.gguf"], "file_integrity": {"granite-switch-4.1-3b-preview-f16.gguf": {"size_bytes": 8428497696, "blob_id": "3c5edd7df00fa175defc73b6f4af58c9b3d66bfc091ade885d3acdd4ded87510"}}, "selector": "F16"}, "execution": {"trunk_layers": 40, "mtp_layers": 0, "activation_width": 2560, "speculative_policy": "mtp-if-present"}, @@ -538,6 +612,7 @@ }, { "family": "grovemoe", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "mradermacher/GroveMoE-Inst-i1-GGUF", "revision": "4e342287d10e5b616e087509fe2a979293bc7b0d", "files": ["GroveMoE-Inst.i1-Q4_K_M.gguf"], "file_integrity": {"GroveMoE-Inst.i1-Q4_K_M.gguf": {"size_bytes": 20167310304, "blob_id": "e1d8ce9e762f74e65d85bea1e8f8e06db70f2dd735cecee9a6d889c71e6fc010"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 48, "mtp_layers": 0, "activation_width": 2048, "speculative_policy": "mtp-if-present"}, @@ -546,6 +621,7 @@ }, { "family": "hunyuan-dense", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "bartowski/tencent_Hunyuan-1.8B-Instruct-GGUF", "revision": "bc16eeb7f561c3d7937aa2cde457668695c237ca", "files": ["tencent_Hunyuan-1.8B-Instruct-Q4_K_M.gguf"], "file_integrity": {"tencent_Hunyuan-1.8B-Instruct-Q4_K_M.gguf": {"size_bytes": 1133084864, "blob_id": "558ae86fc4251fd73cd54685319a83e7550997537e3ad41043ad5c0b5a2a26d4"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 32, "mtp_layers": 0, "activation_width": 2048, "speculative_policy": "mtp-if-present"}, @@ -554,6 +630,7 @@ }, { "family": "hy-v3", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "mradermacher/Tencent-Hy-30B-A3B-uncensored-heretic-i1-GGUF", "revision": "6a7a7d5bea1641ec15202f14d4b467d45c104f06", "files": ["Tencent-Hy-30B-A3B-uncensored-heretic.i1-Q4_K_M.gguf"], "file_integrity": {"Tencent-Hy-30B-A3B-uncensored-heretic.i1-Q4_K_M.gguf": {"size_bytes": 18236703520, "blob_id": "76d19eb68bea4de46f2f3dfc82cfa6896da447b9aec91d7c6c3e7791f2d306ac"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 48, "mtp_layers": 0, "activation_width": 2048, "speculative_policy": "mtp-if-present"}, @@ -562,6 +639,7 @@ }, { "family": "jais", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "mradermacher/jais-family-13b-i1-GGUF", "revision": "0339532e089ec7b6dd502eee16c758fc324046c3", "files": ["jais-family-13b.i1-Q4_K_M.gguf"], "file_integrity": {"jais-family-13b.i1-Q4_K_M.gguf": {"size_bytes": 8967115424, "blob_id": "8783a7d62b714c8520f2b03155f7f52145714dd9006653b7a64a6a6a097f5407"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 40, "mtp_layers": 0, "activation_width": 5120, "speculative_policy": "mtp-if-present"}, @@ -570,6 +648,7 @@ }, { "family": "jais2", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "inception42/Jais-2-8B-Chat-GGUF", "revision": "61365d41aad6a2ffa9d7b7f79dfce21b65833254", "files": ["Q4_K_M.gguf"], "file_integrity": {"Q4_K_M.gguf": {"size_bytes": 5104022560, "blob_id": "26aef0cee0f9960cb12cc74ffa1049ca5e35e8159621aa1bacd9f9f703239923"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 32, "mtp_layers": 0, "activation_width": 3328, "speculative_policy": "mtp-if-present"}, @@ -578,6 +657,7 @@ }, { "family": "kimi-k3", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "murillo2000/Kimi-K3-0.40B-GGUF", "revision": "88de02cf8fa37f87eb06daaed370ac9c3411d5ca", "files": ["Kimi-K3-0.40B-F16.gguf"], "file_integrity": {"Kimi-K3-0.40B-F16.gguf": {"size_bytes": 784318432, "blob_id": "411c197b503e6fb9199a2b22115e32dc4e2cad803fb112b24967737b3bab26c7"}}, "selector": "F16"}, "execution": {"trunk_layers": 8, "mtp_layers": 0, "activation_width": 1024, "speculative_policy": "mtp-if-present"}, @@ -586,6 +666,7 @@ }, { "family": "lfm2-moe", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "noctrex/LFM2.5-8B-A1B-MXFP4_MOE-GGUF", "revision": "6f9e38da4a1715057e3c3a45105037944e767117", "files": ["LFM2.5-8B-A1B-MXFP4_MOE.gguf"], "file_integrity": {"LFM2.5-8B-A1B-MXFP4_MOE.gguf": {"size_bytes": 5138197824, "blob_id": "a492e8b1d8c201b90de421f0ea1b6ed478aa7f4948604ea6659cd9afca715f3a"}}, "selector": "MXFP4_MOE"}, "execution": {"trunk_layers": 24, "mtp_layers": 0, "activation_width": 2048, "speculative_policy": "mtp-if-present"}, @@ -594,6 +675,7 @@ }, { "family": "maincoder", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "mradermacher/Maincoder-1B-GGUF", "revision": "1c963c98dfb478ea3b4719299bd85fbb5cf30899", "files": ["Maincoder-1B.Q4_K_M.gguf"], "file_integrity": {"Maincoder-1B.Q4_K_M.gguf": {"size_bytes": 672108288, "blob_id": "2db4b53955a58f4483442110fdd5534bfa22a45447f35a131b839a034ffa2998"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 32, "mtp_layers": 0, "activation_width": 1536, "speculative_policy": "mtp-if-present"}, @@ -602,6 +684,7 @@ }, { "family": "mellum", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "yuxinlu1/Mellum2-12B-A2.5B-Claude-4.6-4.8-Opus-Thinking-GGUF", "revision": "a9c7b1a32ead28d4a868ff31bbf9c15a97362cfe", "files": ["mellum2-claude-Q4_K_M.gguf"], "file_integrity": {"mellum2-claude-Q4_K_M.gguf": {"size_bytes": 8071294784, "blob_id": "2086900d9374d7a12388b6a44f7d34afcf5817abcba7983dc733d38c5003e2ea"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 28, "mtp_layers": 0, "activation_width": 2304, "speculative_policy": "mtp-if-present"}, @@ -610,6 +693,7 @@ }, { "family": "minicpm", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "openbmb/MiniCPM4.1-8B-GGUF", "revision": "ebb834d0ad9acfa98bd16bc963ec51be5f7e08c1", "files": ["MiniCPM4.1-8B-Q4_K_M.gguf"], "file_integrity": {"MiniCPM4.1-8B-Q4_K_M.gguf": {"size_bytes": 4965526048, "blob_id": "9d2ffb9145bf7a88ddb94b2542b42d925293666c42962aa158eeb62fc9708654"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 32, "mtp_layers": 0, "activation_width": 4096, "speculative_policy": "mtp-if-present"}, @@ -618,6 +702,7 @@ }, { "family": "qwen35moe", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "unsloth/Qwen3.6-35B-A3B-GGUF", "revision": "a483e9e6cbd595906af30beda3187c2663a1118c", "files": ["Qwen3.6-35B-A3B-UD-IQ2_XXS.gguf"], "file_integrity": {"Qwen3.6-35B-A3B-UD-IQ2_XXS.gguf": {"size_bytes": 10756586464, "blob_id": "2e8f5f705355c56311432d0a8a5d14a696dbb7e4b197d05c75ba805fc1857bef"}}, "selector": "UD-IQ2_XXS"}, "execution": {"trunk_layers": 40, "mtp_layers": 0, "activation_width": 2048, "speculative_policy": "mtp-if-present"}, @@ -626,6 +711,7 @@ }, { "family": "qwen3vlmoe", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "Qwen/Qwen3-VL-30B-A3B-Instruct-GGUF", "revision": "f54435e6cc31258f04b0969105c3f6badb197931", "files": ["Qwen3VL-30B-A3B-Instruct-Q4_K_M.gguf"], "file_integrity": {"Qwen3VL-30B-A3B-Instruct-Q4_K_M.gguf": {"size_bytes": 18556687168, "blob_id": "87bb374d849f80ebdfabb304189fac9e0bd35a0f74506e6a59c51b206cbe863b"}}, "selector": "Q4_K_M"}, "mmproj_artifact": {"repo": "Qwen/Qwen3-VL-30B-A3B-Instruct-GGUF", "revision": "f54435e6cc31258f04b0969105c3f6badb197931", "files": ["mmproj-Qwen3VL-30B-A3B-Instruct-Q8_0.gguf"], "file_integrity": {"mmproj-Qwen3VL-30B-A3B-Instruct-Q8_0.gguf": {"size_bytes": 712148928, "blob_id": "82a9966edfdbc1b18a27fd90af96d50cbf111a51484f6290e946dccf461c8150"}}, "selector": "Q8_0"}, @@ -635,6 +721,7 @@ }, { "family": "mistral4", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "unsloth/Mistral-Small-4-119B-2603-GGUF", "revision": "bd93c721735aa32c035c0f19e738cb3371fd56ff", "files": ["Mistral-Small-4-119B-2603-UD-IQ2_XXS.gguf"], "file_integrity": {"Mistral-Small-4-119B-2603-UD-IQ2_XXS.gguf": {"size_bytes": 34924187328, "blob_id": "82a676844e27065abe36cf9292603b6e2484a23ca5665a6d8a0bc62ac478beb3"}}, "selector": "UD-IQ2_XXS"}, "execution": {"trunk_layers": 36, "mtp_layers": 0, "activation_width": 4096, "speculative_policy": "mtp-if-present"}, @@ -643,6 +730,7 @@ }, { "family": "step35", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "bartowski/stepfun-ai_Step-3.5-Flash-GGUF", "revision": "a2943ef56a0a240bdfb9a2feee97a775f612476d", "files": ["stepfun-ai_Step-3.5-Flash-IQ1_S.gguf"], "file_integrity": {"stepfun-ai_Step-3.5-Flash-IQ1_S.gguf": {"size_bytes": 41301534944, "blob_id": "69a6683e8e427c17170aaa55c8ac1160d7b24c18aa2a60ea11e3f5810c3ebb97"}}, "selector": "IQ1_S"}, "execution": {"trunk_layers": 45, "mtp_layers": 0, "activation_width": 4096, "speculative_policy": "mtp-if-present"}, @@ -651,6 +739,7 @@ }, { "family": "mimo2", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "bartowski/MiMo-V2.5-GGUF", "revision": "e528cbf4b6197dd642b393acc069fb1cd529e085", "files": ["MiMo-V2.5-IQ1_S/MiMo-V2.5-IQ1_S-00001-of-00002.gguf", "MiMo-V2.5-IQ1_S/MiMo-V2.5-IQ1_S-00002-of-00002.gguf"], "file_integrity": {"MiMo-V2.5-IQ1_S/MiMo-V2.5-IQ1_S-00001-of-00002.gguf": {"size_bytes": 39894774368, "blob_id": "a4f597f6dd9369240cf12d0b4a8c118e0659bf123b4290e042a6f334c2182520"}, "MiMo-V2.5-IQ1_S/MiMo-V2.5-IQ1_S-00002-of-00002.gguf": {"size_bytes": 24622278784, "blob_id": "50b56e1aabb935f7505b7aea37883957489509bb56c34a575c90496754ad593f"}}, "selector": "IQ1_S"}, "execution": {"trunk_layers": 48, "mtp_layers": 3, "activation_width": 4096, "speculative_policy": "mtp-if-present"}, @@ -659,6 +748,7 @@ }, { "family": "minimax-m3", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "unsloth/MiniMax-M3-GGUF", "revision": "a31ac6425155f773fdde697421c3b8cf42083ecf", "files": ["UD-IQ1_M/MiniMax-M3-UD-IQ1_M-00001-of-00004.gguf", "UD-IQ1_M/MiniMax-M3-UD-IQ1_M-00002-of-00004.gguf", "UD-IQ1_M/MiniMax-M3-UD-IQ1_M-00003-of-00004.gguf", "UD-IQ1_M/MiniMax-M3-UD-IQ1_M-00004-of-00004.gguf"], "file_integrity": {"UD-IQ1_M/MiniMax-M3-UD-IQ1_M-00001-of-00004.gguf": {"size_bytes": 8243104, "blob_id": "b1c2b41d5078f43c2229c790f86266c68ce5d6d23f05c93085951a30acaca998"}, "UD-IQ1_M/MiniMax-M3-UD-IQ1_M-00002-of-00004.gguf": {"size_bytes": 49165334304, "blob_id": "fee0653b3aaaf3f41f0e15c297550995149c4a448b7c7b6d056a6cc227fee3bc"}, "UD-IQ1_M/MiniMax-M3-UD-IQ1_M-00003-of-00004.gguf": {"size_bytes": 49782135840, "blob_id": "c3253c3f5f7d99ab7a6855ce8d1ed870b3630e08cf509dc4606fb5bfdf447f36"}, "UD-IQ1_M/MiniMax-M3-UD-IQ1_M-00004-of-00004.gguf": {"size_bytes": 29461567904, "blob_id": "ac01e73d8d7e829f0627a7a82bb253725f15312f5fec0d00b98ae7e168baafbb"}}, "selector": "UD-IQ1_M"}, "execution": {"trunk_layers": 60, "mtp_layers": 0, "activation_width": 6144, "speculative_policy": "mtp-if-present"}, @@ -667,14 +757,79 @@ }, { "family": "spark2-5", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "XHToken/Spark-X2.5-4B-GGUF", "revision": "d313463f1ff7dc29db20193fddc4ad8386d16261", "files": ["Spark-X2.5-4B-Q4_K_M.gguf"], "file_integrity": {"Spark-X2.5-4B-Q4_K_M.gguf": {"size_bytes": 2600224352, "blob_id": "adfcfa19a4ed6a5985da8bf565fe15f8e1a7e131d79bae2d19d48d1c40109428"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 36, "mtp_layers": 0, "activation_width": 2560, "speculative_policy": "mtp-if-present"}, "resources": {"runner_role": "family-certify", "cache_policy": "immutable-local", "estimated_model_bytes": 2594836480}, "notes": "Spark X2.5 dense mixed-window attention architecture coverage" }, + { + "family": "nomic-bert-embedding", + "class": "embedding", + "profile": "workload-oracle", + "artifact": {"repo": "nomic-ai/nomic-embed-text-v1.5-GGUF", "revision": "0188c9bf409793f810680a5a431e7b899c46104c", "files": ["nomic-embed-text-v1.5.Q8_0.gguf"], "file_integrity": {"nomic-embed-text-v1.5.Q8_0.gguf": {"size_bytes": 146146432, "blob_id": "3e24342164b3d94991ba9692fdc0dd08e3fd7362e0aacc396a9a5c54a544c3b7"}}, "selector": "Q8_0"}, + "evidence": {"fixture": "scripts/workload_fixtures.py#EMBEDDING_INPUTS", "comparison": "batch and individual vectors within 1e-4 coordinate error and 0.99999 cosine"}, + "execution": {"trunk_layers": 12, "mtp_layers": 0, "activation_width": 768, "speculative_policy": "disabled"}, + "resources": {"runner_role": "family-certify", "cache_policy": "immutable-local", "estimated_model_bytes": 145389792}, + "notes": "Local full-model embedding, HTTP and official SDK smoke, and pinned CPU monolithic batch/single-vector parity; staged tensor filtering is intentionally unsupported" + }, + { + "family": "jina-bert-v2-rerank", + "class": "rerank", + "profile": "workload-oracle", + "artifact": {"repo": "ggml-org/jina-reranker-v1-turbo-en-GGUF", "revision": "607d8664c787e517e5d6e339d21f680f9002c931", "files": ["Jina-Bert-Implementation-38M-F16.gguf"], "file_integrity": {"Jina-Bert-Implementation-38M-F16.gguf": {"size_bytes": 76971168, "blob_id": "71abc010bb3dce97812ee971509a5cb6ff6f6b8cfffd8480129242f605521fca"}}, "selector": "F16"}, + "evidence": {"fixture": "scripts/workload_fixtures.py#RERANK_DOCUMENTS", "comparison": "scores within 1e-4 and identical document order"}, + "execution": {"trunk_layers": 6, "mtp_layers": 0, "activation_width": 384, "speculative_policy": "disabled"}, + "resources": {"runner_role": "family-certify", "cache_policy": "immutable-local", "estimated_model_bytes": 75293188}, + "notes": "Local full-model cross-encoder rerank and HTTP smoke with pinned CPU monolithic score/order parity; staged tensor filtering is intentionally unsupported" + }, + { + "family": "t5-encoder-decoder", + "class": "encoder_decoder", + "profile": "workload-oracle", + "artifact": {"repo": "Felladrin/gguf-flan-t5-small", "revision": "d71c51f67519edd3154527c2d8f20288bdde9705", "files": ["flan-t5-small.Q8_0.gguf"], "file_integrity": {"flan-t5-small.Q8_0.gguf": {"size_bytes": 113709824, "blob_id": "f7f769c360b1ba830b10dd3b7e7d146dbcc4d487962be7dd806d7d52e0a9c2f0"}}, "selector": "Q8_0"}, + "evidence": {"fixture": "scripts/workload_fixtures.py#ENCODER_DECODER_PROMPT", "comparison": "identical normalized greedy text from pinned llama-completion"}, + "execution": {"trunk_layers": 8, "mtp_layers": 0, "activation_width": 512, "speculative_policy": "disabled"}, + "resources": {"runner_role": "family-certify", "cache_policy": "immutable-local", "estimated_model_bytes": 112678400}, + "notes": "Local full-model encoder-decoder generation and HTTP smoke with pinned CPU monolithic completion text parity; cross-attention staging is intentionally unsupported" + }, + { + "family": "paddleocr", + "class": "ocr", + "profile": "workload-oracle", + "artifact": {"repo": "PaddlePaddle/PaddleOCR-VL-1.6-GGUF", "revision": "511b09642bb324401f15f97cc23bc67e8f0a291d", "files": ["PaddleOCR-VL-1.6-GGUF.gguf"], "file_integrity": {"PaddleOCR-VL-1.6-GGUF.gguf": {"size_bytes": 935769056, "blob_id": "f3ae46ec885050acf4b3d31944431e1fd90d50664fb09126af4a3c050ba14ee8"}}, "selector": "BF16"}, + "mmproj_artifact": {"repo": "PaddlePaddle/PaddleOCR-VL-1.6-GGUF", "revision": "511b09642bb324401f15f97cc23bc67e8f0a291d", "files": ["PaddleOCR-VL-1.6-GGUF-mmproj.gguf"], "file_integrity": {"PaddleOCR-VL-1.6-GGUF-mmproj.gguf": {"size_bytes": 881770560, "blob_id": "204d757d7610d9b3faab10d506d69e5b244e32bf765e2bab2d0167e65e0a058a"}}, "selector": "BF16"}, + "evidence": {"fixture": "scripts/generate-ocr-oracle-fixture.py#MESH 42", "comparison": "normalized monolithic text parity plus independent MESH 42 label"}, + "execution": {"trunk_layers": 18, "mtp_layers": 0, "activation_width": 1024, "speculative_policy": "disabled"}, + "resources": {"runner_role": "family-certify", "cache_policy": "immutable-local", "estimated_model_bytes": 933384192, "startup_timeout_secs": 900}, + "notes": "Local full-model OCR and HTTP smoke with the pinned PaddleOCR projector and mandatory generated-image monolithic parity; projector and trunk remain colocated" + }, + { + "family": "qwen3tts", + "class": "speech_synthesis", + "profile": "workload-oracle", + "artifact": {"repo": "ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF", "revision": "ca27d74bc954b73dadab5b71ca265d87fc861a7c", "files": ["Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf"], "file_integrity": {"Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf": {"size_bytes": 1847874400, "blob_id": "ac7931aeb2e7aad1a6ed6602d353a5679c9d096b18ce8204ac730a8408d572e1"}}, "selector": "Q8_0"}, + "mmproj_artifact": {"repo": "ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF", "revision": "ca27d74bc954b73dadab5b71ca265d87fc861a7c", "files": ["mmproj-Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf"], "file_integrity": {"mmproj-Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf": {"size_bytes": 446422912, "blob_id": "6fd65188839bcd6ecc91b277ad471e22a0edfada4699a0fe82f1165c18cfcce2"}}, "selector": "Q8_0"}, + "evidence": {"fixture": "scripts/skippy-tts-oracle.py#PROMPT", "comparison": "fixed-seed PCM format and length; RMS error at most 2 percent and waveform cosine at least 0.9995"}, + "execution": {"trunk_layers": 28, "mtp_layers": 0, "activation_width": 2048, "speculative_policy": "disabled"}, + "resources": {"runner_role": "family-certify", "cache_policy": "immutable-local", "estimated_model_bytes": 1841844224, "startup_timeout_secs": 600}, + "notes": "Local full-model speech-synthesis and HTTP WAV smoke with deterministic pinned monolithic PCM parity; distributed staging is intentionally unsupported" + }, + { + "family": "ultravox", + "class": "speech_recognition", + "profile": "workload-oracle", + "artifact": {"repo": "ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF", "revision": "5390c7c41cbd6f261f7f205fc0c5ae61bbdca650", "files": ["Llama-3.2-1B-Instruct-Q8_0.gguf"], "file_integrity": {"Llama-3.2-1B-Instruct-Q8_0.gguf": {"size_bytes": 1321083008, "blob_id": "432f310a77f4650a88d0fd59ecdd7cebed8d684bafea53cbff0473542964f0c3"}}, "selector": "Q8_0"}, + "mmproj_artifact": {"repo": "ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF", "revision": "5390c7c41cbd6f261f7f205fc0c5ae61bbdca650", "files": ["mmproj-ultravox-v0_5-llama-3_2-1b-f16.gguf"], "file_integrity": {"mmproj-ultravox-v0_5-llama-3_2-1b-f16.gguf": {"size_bytes": 1371123616, "blob_id": "b34dde1835752949d6b960528269af93c92fec91c61ea0534fcc73f96c1ed8b2"}}, "selector": "F16"}, + "evidence": {"fixture": "ci/llama-canary/fixtures/audio-smoke.wav", "comparison": "identical normalized transcript against pinned llama-server; fixture is unlabeled"}, + "execution": {"trunk_layers": 16, "mtp_layers": 0, "activation_width": 2048, "speculative_policy": "disabled"}, + "resources": {"runner_role": "family-certify", "cache_policy": "immutable-local", "estimated_model_bytes": 1313251456, "startup_timeout_secs": 600}, + "notes": "Local full-model audio-to-text and HTTP smoke with the pinned Ultravox projector and monolithic transcript parity; unlabeled WAV does not establish transcription accuracy" + }, { "family": "inkling", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "unsloth/Inkling-Small-GGUF", "revision": "1a19ef82883cb7b9c581b93c30ea252dabbf658d", "files": ["UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00001-of-00005.gguf", "UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00002-of-00005.gguf", "UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00003-of-00005.gguf", "UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00004-of-00005.gguf", "UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00005-of-00005.gguf"], "file_integrity": {"UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00001-of-00005.gguf": {"size_bytes": 12986592, "blob_id": "a51ac3f439198f2817219edd582be4b600c273be24e78cbd58ebff982d9f007e"}, "UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00002-of-00005.gguf": {"size_bytes": 48810843872, "blob_id": "3dccdd473cc3a191e6028f6105b01831ed3a8dc30ec4e02679f9e0c2ccb59671"}, "UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00003-of-00005.gguf": {"size_bytes": 49237390240, "blob_id": "1a7edf29bda1d278b4668e1a082d7db634b845a53ea58c918a8cea1f9006c21c"}, "UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00004-of-00005.gguf": {"size_bytes": 49458817088, "blob_id": "376f67568438da96b10566730e8a9e17e3f665ab1b9d82eba48ec33708b172f7"}, "UD-Q4_K_M/Inkling-Small-UD-Q4_K_M-00005-of-00005.gguf": {"size_bytes": 15017569600, "blob_id": "e34364af0d04d2d295bc374f1a4fa80002e4277db1bcf4b56240dbd443ac21d3"}}, "selector": "UD-Q4_K_M"}, "mmproj_artifact": {"repo": "unsloth/Inkling-Small-GGUF", "revision": "1a19ef82883cb7b9c581b93c30ea252dabbf658d", "files": ["mmproj-F16.gguf"], "file_integrity": {"mmproj-F16.gguf": {"size_bytes": 138684128, "blob_id": "bf1e6afc9889151b3fee61119d3ee2cd224b59a1d881cdf87e33962f64aecc7e"}}, "selector": "F16"}, @@ -684,6 +839,7 @@ }, { "family": "llama4", + "class": "causal_generation", "profile": "full", "artifact": {"repo": "unsloth/Llama-4-Scout-17B-16E-Instruct-GGUF", "revision": "72a6853f56a66dc13a3a4b6bdc9cf7ee4c364b47", "files": ["Q4_K_M/Llama-4-Scout-17B-16E-Instruct-Q4_K_M-00001-of-00002.gguf", "Q4_K_M/Llama-4-Scout-17B-16E-Instruct-Q4_K_M-00002-of-00002.gguf"], "file_integrity": {"Q4_K_M/Llama-4-Scout-17B-16E-Instruct-Q4_K_M-00001-of-00002.gguf": {"size_bytes": 49848379744, "blob_id": "fbe956902467171ed7c0c326e5d868771a84d46468d407abecd0f289297313f9"}, "Q4_K_M/Llama-4-Scout-17B-16E-Instruct-Q4_K_M-00002-of-00002.gguf": {"size_bytes": 15511520608, "blob_id": "e7330ae14527f08e8a44a6a573b45084052b8822c4b8a5179c5cad9cd6f6f795"}}, "selector": "Q4_K_M"}, "execution": {"trunk_layers": 48, "mtp_layers": 0, "activation_width": 5120, "speculative_policy": "mtp-if-present"}, diff --git a/ci/llama-canary/family-certified.schema.json b/ci/llama-canary/family-certified.schema.json index c89a3c6ffc..eca1dcaffd 100644 --- a/ci/llama-canary/family-certified.schema.json +++ b/ci/llama-canary/family-certified.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://mesh-llm.dev/schemas/family-certified.schema.json", "title": "MeshLLM supported-family certification policy", - "$comment": "The schema validates transport shape. scripts/plan-family-battery.py additionally enforces the three-lane certified-profile invariant, exact profile semantics, unique families, safe artifact paths, and deterministic sharding.", + "$comment": "The schema validates transport shape and workload/profile separation. scripts/plan-family-battery.py additionally enforces exact class-specific lane semantics, unique families, safe artifact paths, and deterministic sharding.", "type": "object", "additionalProperties": false, "required": ["schema_version", "policy", "models"], @@ -16,11 +16,13 @@ "profiles": { "type": "object", "additionalProperties": false, - "required": ["full", "package-oracle", "graph-only"], + "required": ["full", "package-oracle", "graph-only", "workload-smoke", "workload-oracle"], "properties": { "full": {"$ref": "#/$defs/profile"}, "package-oracle": {"$ref": "#/$defs/profile"}, - "graph-only": {"$ref": "#/$defs/profile"} + "graph-only": {"$ref": "#/$defs/profile"}, + "workload-smoke": {"$ref": "#/$defs/profile"}, + "workload-oracle": {"$ref": "#/$defs/profile"} } } } @@ -86,6 +88,7 @@ "additionalProperties": false, "required": [ "family", + "class", "profile", "artifact", "execution", @@ -94,7 +97,17 @@ ], "properties": { "family": {"type": "string", "pattern": "^[a-z0-9][a-z0-9._-]*$"}, - "profile": {"enum": ["full", "package-oracle", "graph-only"]}, + "class": {"enum": ["causal_generation", "embedding", "rerank", "encoder_decoder", "ocr", "speech_synthesis", "speech_recognition"]}, + "profile": {"enum": ["full", "package-oracle", "graph-only", "workload-smoke", "workload-oracle"]}, + "evidence": { + "type": "object", + "additionalProperties": false, + "required": ["fixture", "comparison"], + "properties": { + "fixture": {"$ref": "#/$defs/singleLine"}, + "comparison": {"$ref": "#/$defs/singleLine"} + } + }, "artifact": {"$ref": "#/$defs/artifact"}, "draft_artifact": {"$ref": "#/$defs/artifact"}, "mmproj_artifact": {"$ref": "#/$defs/artifact"}, @@ -126,7 +139,29 @@ } }, "notes": {"$ref": "#/$defs/singleLine"} - } + }, + "allOf": [ + { + "if": {"properties": {"class": {"const": "causal_generation"}}}, + "then": {"properties": {"profile": {"enum": ["full", "package-oracle", "graph-only"]}}}, + "else": { + "properties": { + "profile": {"enum": ["workload-smoke", "workload-oracle"]}, + "artifact": {"properties": {"files": {"maxItems": 1}}}, + "execution": {"properties": {"mtp_layers": {"const": 0}, "speculative_policy": {"const": "disabled"}}} + } + } + }, + { + "if": {"properties": {"profile": {"const": "workload-oracle"}}}, + "then": {"required": ["evidence"]}, + "else": {"not": {"required": ["evidence"]}} + }, + { + "if": {"properties": {"class": {"enum": ["ocr", "speech_synthesis", "speech_recognition"]}}}, + "then": {"required": ["mmproj_artifact"]} + } + ] } } } diff --git a/ci/llama-canary/fixtures/audio-smoke.wav b/ci/llama-canary/fixtures/audio-smoke.wav new file mode 100644 index 0000000000..41effdd9be Binary files /dev/null and b/ci/llama-canary/fixtures/audio-smoke.wav differ diff --git a/ci/model-artifacts/manifests/competitive-benchmark.json b/ci/model-artifacts/manifests/competitive-benchmark.json index 615fe417a3..4d6430a8a2 100644 --- a/ci/model-artifacts/manifests/competitive-benchmark.json +++ b/ci/model-artifacts/manifests/competitive-benchmark.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "competitive-benchmark", - "registry_sha256": "b385a5723495b90b15c27b666b2907f6fb94aa76dabeb927a40b2d6f01951aad", + "registry_sha256": "974ae22d387387ded978bd3d0c5e50c6a817231622d7f388df6c1aeca8ae9940", "artifacts": [ { "id": "family-llama", diff --git a/ci/model-artifacts/manifests/hf-download-smoke.json b/ci/model-artifacts/manifests/hf-download-smoke.json index 61d53795d0..cda3b27c31 100644 --- a/ci/model-artifacts/manifests/hf-download-smoke.json +++ b/ci/model-artifacts/manifests/hf-download-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "hf-download-smoke", - "registry_sha256": "b385a5723495b90b15c27b666b2907f6fb94aa76dabeb927a40b2d6f01951aad", + "registry_sha256": "974ae22d387387ded978bd3d0c5e50c6a817231622d7f388df6c1aeca8ae9940", "artifacts": [ { "id": "smollm2-q4-download", diff --git a/ci/model-artifacts/manifests/openai-smoke.json b/ci/model-artifacts/manifests/openai-smoke.json index 66641a1868..12df367135 100644 --- a/ci/model-artifacts/manifests/openai-smoke.json +++ b/ci/model-artifacts/manifests/openai-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "openai-smoke", - "registry_sha256": "b385a5723495b90b15c27b666b2907f6fb94aa76dabeb927a40b2d6f01951aad", + "registry_sha256": "974ae22d387387ded978bd3d0c5e50c6a817231622d7f388df6c1aeca8ae9940", "artifacts": [ { "id": "smollm2-q4-download", diff --git a/ci/model-artifacts/manifests/product-integration-smoke.json b/ci/model-artifacts/manifests/product-integration-smoke.json index 91cf02ae60..a4605c1129 100644 --- a/ci/model-artifacts/manifests/product-integration-smoke.json +++ b/ci/model-artifacts/manifests/product-integration-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "product-integration-smoke", - "registry_sha256": "b385a5723495b90b15c27b666b2907f6fb94aa76dabeb927a40b2d6f01951aad", + "registry_sha256": "974ae22d387387ded978bd3d0c5e50c6a817231622d7f388df6c1aeca8ae9940", "artifacts": [ { "id": "family-granite-hybrid", diff --git a/ci/model-artifacts/manifests/product-smoke.json b/ci/model-artifacts/manifests/product-smoke.json index 7d0c3f402f..4f35cb7943 100644 --- a/ci/model-artifacts/manifests/product-smoke.json +++ b/ci/model-artifacts/manifests/product-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "product-smoke", - "registry_sha256": "b385a5723495b90b15c27b666b2907f6fb94aa76dabeb927a40b2d6f01951aad", + "registry_sha256": "974ae22d387387ded978bd3d0c5e50c6a817231622d7f388df6c1aeca8ae9940", "artifacts": [ { "id": "smollm2-q8-inference", diff --git a/ci/model-artifacts/manifests/radix-cache.json b/ci/model-artifacts/manifests/radix-cache.json index 594faff476..3909162533 100644 --- a/ci/model-artifacts/manifests/radix-cache.json +++ b/ci/model-artifacts/manifests/radix-cache.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "radix-cache", - "registry_sha256": "b385a5723495b90b15c27b666b2907f6fb94aa76dabeb927a40b2d6f01951aad", + "registry_sha256": "974ae22d387387ded978bd3d0c5e50c6a817231622d7f388df6c1aeca8ae9940", "artifacts": [ { "id": "family-deepseek2", diff --git a/ci/model-artifacts/manifests/safetensors-runtime-smoke.json b/ci/model-artifacts/manifests/safetensors-runtime-smoke.json index 655197d871..260267caaf 100644 --- a/ci/model-artifacts/manifests/safetensors-runtime-smoke.json +++ b/ci/model-artifacts/manifests/safetensors-runtime-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "safetensors-runtime-smoke", - "registry_sha256": "b385a5723495b90b15c27b666b2907f6fb94aa76dabeb927a40b2d6f01951aad", + "registry_sha256": "974ae22d387387ded978bd3d0c5e50c6a817231622d7f388df6c1aeca8ae9940", "artifacts": [ { "id": "smollm2-safetensors", diff --git a/ci/model-artifacts/manifests/scripted-binary-smoke.json b/ci/model-artifacts/manifests/scripted-binary-smoke.json index 2af301f84e..7e092c9a78 100644 --- a/ci/model-artifacts/manifests/scripted-binary-smoke.json +++ b/ci/model-artifacts/manifests/scripted-binary-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "scripted-binary-smoke", - "registry_sha256": "b385a5723495b90b15c27b666b2907f6fb94aa76dabeb927a40b2d6f01951aad", + "registry_sha256": "974ae22d387387ded978bd3d0c5e50c6a817231622d7f388df6c1aeca8ae9940", "artifacts": [ { "id": "smollm2-q8-inference", diff --git a/ci/model-artifacts/manifests/sdk-smoke.json b/ci/model-artifacts/manifests/sdk-smoke.json index 4db0f7edae..a0f812be87 100644 --- a/ci/model-artifacts/manifests/sdk-smoke.json +++ b/ci/model-artifacts/manifests/sdk-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "sdk-smoke", - "registry_sha256": "b385a5723495b90b15c27b666b2907f6fb94aa76dabeb927a40b2d6f01951aad", + "registry_sha256": "974ae22d387387ded978bd3d0c5e50c6a817231622d7f388df6c1aeca8ae9940", "artifacts": [ { "id": "smollm2-q8-inference", diff --git a/ci/model-artifacts/manifests/skippy-ci-smoke.json b/ci/model-artifacts/manifests/skippy-ci-smoke.json index d26f8977ce..ec8be0b411 100644 --- a/ci/model-artifacts/manifests/skippy-ci-smoke.json +++ b/ci/model-artifacts/manifests/skippy-ci-smoke.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "skippy-ci-smoke", - "registry_sha256": "b385a5723495b90b15c27b666b2907f6fb94aa76dabeb927a40b2d6f01951aad", + "registry_sha256": "974ae22d387387ded978bd3d0c5e50c6a817231622d7f388df6c1aeca8ae9940", "artifacts": [ { "id": "family-qwen3-dense", diff --git a/ci/model-artifacts/manifests/skippy-correctness.json b/ci/model-artifacts/manifests/skippy-correctness.json index 47b440fac0..4943c48616 100644 --- a/ci/model-artifacts/manifests/skippy-correctness.json +++ b/ci/model-artifacts/manifests/skippy-correctness.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "skippy-correctness", - "registry_sha256": "b385a5723495b90b15c27b666b2907f6fb94aa76dabeb927a40b2d6f01951aad", + "registry_sha256": "974ae22d387387ded978bd3d0c5e50c6a817231622d7f388df6c1aeca8ae9940", "artifacts": [ { "id": "qwen3-q8-correctness", diff --git a/ci/model-artifacts/manifests/skippy-parity.json b/ci/model-artifacts/manifests/skippy-parity.json index e4d4030ee8..703c7d7d40 100644 --- a/ci/model-artifacts/manifests/skippy-parity.json +++ b/ci/model-artifacts/manifests/skippy-parity.json @@ -2,7 +2,7 @@ "schema_version": 1, "manifest_kind": "test-model-artifacts", "suite": "skippy-parity", - "registry_sha256": "b385a5723495b90b15c27b666b2907f6fb94aa76dabeb927a40b2d6f01951aad", + "registry_sha256": "974ae22d387387ded978bd3d0c5e50c6a817231622d7f388df6c1aeca8ae9940", "artifacts": [ { "id": "family-deepseek2", diff --git a/ci/model-artifacts/registry.json b/ci/model-artifacts/registry.json index 28510b9ede..5a26a329c5 100644 --- a/ci/model-artifacts/registry.json +++ b/ci/model-artifacts/registry.json @@ -53,6 +53,21 @@ "tensor-ownership", "stage-load" ] + }, + "workload-smoke": { + "status": "provisional", + "oracle": "none", + "required_lanes": [ + "class-specific-smoke" + ] + }, + "workload-oracle": { + "status": "certified", + "oracle": "local-monolithic", + "required_lanes": [ + "class-specific-smoke", + "class-specific-oracle" + ] } } }, @@ -116,7 +131,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 633495552 }, - "notes": "core production dense baseline" + "notes": "core production dense baseline", + "class": "causal_generation" } }, { @@ -162,7 +178,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 799862912 }, - "notes": "production dense baseline; covers the trimmed olmo2 architecture path" + "notes": "production dense baseline; covers the trimmed olmo2 architecture path", + "class": "causal_generation" } }, { @@ -205,7 +222,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 18302863872 }, - "notes": "production MLA path shared with deepseek2; tensors span blk.0 through blk.46" + "notes": "production MLA path shared with deepseek2; tensors span blk.0 through blk.46", + "class": "causal_generation" } }, { @@ -247,7 +265,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 6244892672 }, - "notes": "production dense GLM path" + "notes": "production dense GLM path", + "class": "causal_generation" } }, { @@ -296,7 +315,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 72966388736 }, - "notes": "GLM4.5 MoE lineage; 46 trunk layers plus one MTP runtime layer; two shards" + "notes": "GLM4.5 MoE lineage; 46 trunk layers plus one MTP runtime layer; two shards", + "class": "causal_generation" } }, { @@ -342,7 +362,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 10360420352 }, - "notes": "production MLA path" + "notes": "production MLA path", + "class": "causal_generation" } }, { @@ -385,7 +406,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 12096558336 }, - "notes": "MoE plus MXFP4; only quantization shipped" + "notes": "MoE plus MXFP4; only quantization shipped", + "class": "causal_generation" } }, { @@ -427,7 +449,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 18550716416 }, - "notes": "production MoE; graph_max_nodes budget lane" + "notes": "production MoE; graph_max_nodes budget lane", + "class": "causal_generation" } }, { @@ -469,7 +492,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 9490282496 }, - "notes": "small MoE architecture probe" + "notes": "small MoE architecture probe", + "class": "causal_generation" } }, { @@ -518,7 +542,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 942121232 }, - "notes": "hybrid boundary sweep" + "notes": "hybrid boundary sweep", + "class": "causal_generation" } }, { @@ -561,7 +586,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 1931112448 }, - "notes": "hybrid boundary sweep" + "notes": "hybrid boundary sweep", + "class": "causal_generation" } }, { @@ -605,7 +631,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 13890288384 }, - "notes": "hybrid 3:1 interleave; canonical boundary-sweep model" + "notes": "hybrid 3:1 interleave; canonical boundary-sweep model", + "class": "causal_generation" } }, { @@ -650,7 +677,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 22211435520 }, - "notes": "hybrid three-linear-to-one-attention path" + "notes": "hybrid three-linear-to-one-attention path", + "class": "causal_generation" } }, { @@ -705,7 +733,8 @@ "estimated_model_bytes": 72535436800, "startup_timeout_secs": 900 }, - "notes": "Qwen4 experimental hybrid path; four-wide hyper-connected residual boundary and QSA indexer" + "notes": "Qwen4 experimental hybrid path; four-wide hyper-connected residual boundary and QSA indexer", + "class": "causal_generation" } }, { @@ -750,7 +779,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 25258358528 }, - "notes": "hybrid Mamba2 plus attention MoE; replacement for the invalid all-zero head_count_kv artifact" + "notes": "hybrid Mamba2 plus attention MoE; replacement for the invalid all-zero head_count_kv artifact", + "class": "causal_generation" } }, { @@ -793,7 +823,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 101794800 }, - "notes": "cheap recurrent boundary sweep" + "notes": "cheap recurrent boundary sweep", + "class": "causal_generation" } }, { @@ -835,7 +866,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 4201906176 }, - "notes": "Mamba2 SSM canonical path" + "notes": "Mamba2 SSM canonical path", + "class": "causal_generation" } }, { @@ -877,7 +909,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 991330304 }, - "notes": "recurrent v_first sideband crossing" + "notes": "recurrent v_first sideband crossing", + "class": "causal_generation" } }, { @@ -919,7 +952,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 2313461760 }, - "notes": "recurrent; only real RWKV7 World3 GGUF" + "notes": "recurrent; only real RWKV7 World3 GGUF", + "class": "causal_generation" } }, { @@ -965,7 +999,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 219091712 }, - "notes": "cheap hybrid boundary sweep" + "notes": "cheap hybrid boundary sweep", + "class": "causal_generation" } }, { @@ -1008,7 +1043,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 521555200 }, - "notes": "smallest public Qwen3.5 dense GGUF; hybrid gated-delta-net family with full-attention interval 4; coverage-expansion repair from the failed CI fixture row" + "notes": "smallest public Qwen3.5 dense GGUF; hybrid gated-delta-net family with full-attention interval 4; coverage-expansion repair from the failed CI fixture row", + "class": "causal_generation" } }, { @@ -1050,7 +1086,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 1543549872 }, - "notes": "dense Granite path" + "notes": "dense Granite path", + "class": "causal_generation" } }, { @@ -1093,7 +1130,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 226925568 }, - "notes": "cheap hybrid boundary sweep" + "notes": "cheap hybrid boundary sweep", + "class": "causal_generation" } }, { @@ -1150,7 +1188,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 376830976 }, - "notes": "small official multimodal hybrid model; text, recurrent, and mmproj lanes" + "notes": "small official multimodal hybrid model; text, recurrent, and mmproj lanes", + "class": "causal_generation" } }, { @@ -1192,7 +1231,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 20270574592 }, - "notes": "dense explicit poolside row" + "notes": "dense explicit poolside row", + "class": "causal_generation" } }, { @@ -1247,7 +1287,8 @@ "sha256": "05cc3ae461a7b6aa4023312ccab549ecab77cf8677efee04f049fcbab55b8bc3" } ] - } + }, + "class": "causal_generation" } }, { @@ -1303,7 +1344,8 @@ "sha256": "d406d03ebabefdef86a2c86bf0c1b65f9e046f7a81c218f25de4931b46a07fc4" } ] - } + }, + "class": "causal_generation" } }, { @@ -1345,7 +1387,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 10457367552 }, - "notes": "dense; only file in repository; gemma3 path trimmed" + "notes": "dense; only file in repository; gemma3 path trimmed", + "class": "causal_generation" } }, { @@ -1387,7 +1430,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 2392493568 }, - "notes": "dense Phi3 path" + "notes": "dense Phi3 path", + "class": "causal_generation" } }, { @@ -1429,7 +1473,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 8886743040 }, - "notes": "dense; Microsoft repository does not publish Q4_K_M" + "notes": "dense; Microsoft repository does not publish Q4_K_M", + "class": "causal_generation" } }, { @@ -1471,7 +1516,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 4972826368 }, - "notes": "dense Falcon path" + "notes": "dense Falcon path", + "class": "causal_generation" } }, { @@ -1513,7 +1559,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 3778420736 }, - "notes": "dense InternLM2 path" + "notes": "dense InternLM2 path", + "class": "causal_generation" } }, { @@ -1555,7 +1602,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 14326026240 }, - "notes": "dense; relabelled from mistral3" + "notes": "dense; relabelled from mistral3", + "class": "causal_generation" } }, { @@ -1597,7 +1645,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 5046091776 }, - "notes": "dense Cohere2 path" + "notes": "dense Cohere2 path", + "class": "causal_generation" } }, { @@ -1639,7 +1688,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 1907462144 }, - "notes": "dense SmolLM3 path" + "notes": "dense SmolLM3 path", + "class": "causal_generation" } }, { @@ -1831,7 +1881,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 391859712 }, - "notes": "expansion: dense qwen2 arch coverage" + "notes": "expansion: dense qwen2 arch coverage", + "class": "causal_generation" } }, { @@ -1873,7 +1924,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 548823040 }, - "notes": "expansion: bloom arch coverage" + "notes": "expansion: bloom arch coverage", + "class": "causal_generation" } }, { @@ -1915,7 +1967,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 1489203200 }, - "notes": "expansion: dense gemma arch coverage" + "notes": "expansion: dense gemma arch coverage", + "class": "causal_generation" } }, { @@ -1957,7 +2010,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 799525120 }, - "notes": "expansion: dense gemma3 arch coverage" + "notes": "expansion: dense gemma3 arch coverage", + "class": "causal_generation" } }, { @@ -1999,7 +2053,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 1787432960 }, - "notes": "expansion: dense phi2 arch coverage" + "notes": "expansion: dense phi2 arch coverage", + "class": "causal_generation" } }, { @@ -2041,7 +2096,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 1847230464 }, - "notes": "expansion: dense starcoder2 arch coverage" + "notes": "expansion: dense starcoder2 arch coverage", + "class": "causal_generation" } }, { @@ -2083,7 +2139,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 1027883008 }, - "notes": "expansion: dense stablelm arch coverage" + "notes": "expansion: dense stablelm arch coverage", + "class": "causal_generation" } }, { @@ -2125,7 +2182,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 931962880 }, - "notes": "expansion: dense olmo2 arch coverage" + "notes": "expansion: dense olmo2 arch coverage", + "class": "causal_generation" } }, { @@ -2167,7 +2225,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 808230016 }, - "notes": "expansion: dense exaone4 arch coverage" + "notes": "expansion: dense exaone4 arch coverage", + "class": "causal_generation" } }, { @@ -2209,7 +2268,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 2468069280 }, - "notes": "expansion: dense minicpm3 arch coverage" + "notes": "expansion: dense minicpm3 arch coverage", + "class": "causal_generation" } }, { @@ -2251,7 +2311,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 2691047424 }, - "notes": "expansion: dense nemotron arch coverage (distinct from nemotron_h_moe row)" + "notes": "expansion: dense nemotron arch coverage (distinct from nemotron_h_moe row)", + "class": "causal_generation" } }, { @@ -2293,7 +2354,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 2908474660 }, - "notes": "expansion: arcee/AFM arch coverage" + "notes": "expansion: arcee/AFM arch coverage", + "class": "causal_generation" } }, { @@ -2335,7 +2397,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 4388741120 }, - "notes": "expansion: mpt arch coverage" + "notes": "expansion: mpt arch coverage", + "class": "causal_generation" } }, { @@ -2377,7 +2440,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 5049991424 }, - "notes": "expansion: apertus arch coverage" + "notes": "expansion: apertus arch coverage", + "class": "causal_generation" } }, { @@ -2419,7 +2483,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 6157238272 }, - "notes": "expansion: glm4/chatglm arch coverage (distinct from existing glm4 row arch chatglm: replaces it only if identical family label is rejected)" + "notes": "expansion: glm4/chatglm arch coverage (distinct from existing glm4 row arch chatglm: replaces it only if identical family label is rejected)", + "class": "causal_generation" } }, { @@ -2461,7 +2526,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 15815687168 }, - "notes": "downloaded AFMoE architecture coverage" + "notes": "downloaded AFMoE architecture coverage", + "class": "causal_generation" } }, { @@ -2503,7 +2569,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 5111586816 }, - "notes": "ARWKV recurrent architecture coverage" + "notes": "ARWKV recurrent architecture coverage", + "class": "causal_generation" } }, { @@ -2547,7 +2614,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 15796958848 }, - "notes": "hybrid three-recurrent-to-one-attention boundary sweep" + "notes": "hybrid three-recurrent-to-one-attention boundary sweep", + "class": "causal_generation" } }, { @@ -2589,7 +2657,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 4272308224 }, - "notes": "downloaded Chameleon architecture coverage" + "notes": "downloaded Chameleon architecture coverage", + "class": "causal_generation" } }, { @@ -2632,7 +2701,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 16125871104 }, - "notes": "compact four-layer DeepSeek 3.2 architecture probe" + "notes": "compact four-layer DeepSeek 3.2 architecture probe", + "class": "causal_generation" } }, { @@ -2674,7 +2744,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 14296595968 }, - "notes": "downloaded dots1 MoE architecture coverage" + "notes": "downloaded dots1 MoE architecture coverage", + "class": "causal_generation" } }, { @@ -2716,7 +2787,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 238262272 }, - "notes": "small ERNIE 4.5 dense architecture probe" + "notes": "small ERNIE 4.5 dense architecture probe", + "class": "causal_generation" } }, { @@ -2758,7 +2830,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 13243429632 }, - "notes": "downloaded ERNIE 4.5 MoE architecture coverage" + "notes": "downloaded ERNIE 4.5 MoE architecture coverage", + "class": "causal_generation" } }, { @@ -2800,7 +2873,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 4230498240 }, - "notes": "text-only Gemma 3n architecture coverage" + "notes": "text-only Gemma 3n architecture coverage", + "class": "causal_generation" } }, { @@ -2854,7 +2928,8 @@ "sha256": "debad39ab9c1152ab67695a674fb35e8375b2320c57bfd5075835d3ccb16c7db" } ] - } + }, + "class": "causal_generation" } }, { @@ -2896,7 +2971,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 13133230080 }, - "notes": "downloaded GPT-NeoX architecture coverage" + "notes": "downloaded GPT-NeoX architecture coverage", + "class": "causal_generation" } }, { @@ -2939,7 +3015,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 6044351680 }, - "notes": "Granite sliding-window MoE architecture coverage" + "notes": "Granite sliding-window MoE architecture coverage", + "class": "causal_generation" } }, { @@ -2981,7 +3058,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 820107736 }, - "notes": "downloaded Granite MoE architecture coverage" + "notes": "downloaded Granite MoE architecture coverage", + "class": "causal_generation" } }, { @@ -3023,7 +3101,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 8424888320 }, - "notes": "Granite Switch adapter-router architecture coverage" + "notes": "Granite Switch adapter-router architecture coverage", + "class": "causal_generation" } }, { @@ -3065,7 +3144,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 20161329152 }, - "notes": "downloaded GroveMoE architecture coverage" + "notes": "downloaded GroveMoE architecture coverage", + "class": "causal_generation" } }, { @@ -3107,7 +3187,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 1127990304 }, - "notes": "downloaded Hunyuan dense architecture coverage" + "notes": "downloaded Hunyuan dense architecture coverage", + "class": "causal_generation" } }, { @@ -3149,7 +3230,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 18231574016 }, - "notes": "downloaded Tencent HY v3 MoE architecture coverage" + "notes": "downloaded Tencent HY v3 MoE architecture coverage", + "class": "causal_generation" } }, { @@ -3191,7 +3273,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 8962816000 }, - "notes": "downloaded Jais architecture coverage" + "notes": "downloaded Jais architecture coverage", + "class": "causal_generation" } }, { @@ -3233,7 +3316,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 5097211392 }, - "notes": "gated Jais 2 artifact with locally verified immutable digest" + "notes": "gated Jais 2 artifact with locally verified immutable digest", + "class": "causal_generation" } }, { @@ -3277,7 +3361,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 777396384 }, - "notes": "compact hybrid three-recurrent-to-one-attention boundary sweep" + "notes": "compact hybrid three-recurrent-to-one-attention boundary sweep", + "class": "causal_generation" } }, { @@ -3322,7 +3407,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 5129959168 }, - "notes": "LFM2 MoE hybrid recurrent boundary sweep" + "notes": "LFM2 MoE hybrid recurrent boundary sweep", + "class": "causal_generation" } }, { @@ -3364,7 +3450,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 666155520 }, - "notes": "downloaded Maincoder architecture coverage" + "notes": "downloaded Maincoder architecture coverage", + "class": "causal_generation" } }, { @@ -3407,7 +3494,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 8067705856 }, - "notes": "Mellum MoE sliding-window architecture coverage" + "notes": "Mellum MoE sliding-window architecture coverage", + "class": "causal_generation" } }, { @@ -3449,7 +3537,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 4963831552 }, - "notes": "downloaded MiniCPM 4.1 architecture coverage" + "notes": "downloaded MiniCPM 4.1 architecture coverage", + "class": "causal_generation" } }, { @@ -3493,7 +3582,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 10745596416 }, - "notes": "Qwen3.6 MoE gated-delta-net family with a four-layer full-attention interval" + "notes": "Qwen3.6 MoE gated-delta-net family with a four-layer full-attention interval", + "class": "causal_generation" } }, { @@ -3550,7 +3640,8 @@ "sha256": "82a9966edfdbc1b18a27fd90af96d50cbf111a51484f6290e946dccf461c8150" } ] - } + }, + "class": "causal_generation" } }, { @@ -3593,7 +3684,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 34916274176 }, - "notes": "Mistral Small 4 MoE and MLA architecture coverage" + "notes": "Mistral Small 4 MoE and MLA architecture coverage", + "class": "causal_generation" } }, { @@ -3634,7 +3726,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 41296256512 }, - "notes": "Step 3.5 Flash MoE with three leading dense blocks and mixed attention windows" + "notes": "Step 3.5 Flash MoE with three leading dense blocks and mixed attention windows", + "class": "causal_generation" } }, { @@ -3682,7 +3775,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 64511067648 }, - "notes": "MiMo V2.5 MoE with three native NextN heads" + "notes": "MiMo V2.5 MoE with three native NextN heads", + "class": "causal_generation" } }, { @@ -3739,7 +3833,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 128408977920 }, - "notes": "MiniMax M3 MoE with three leading dense blocks and sparse-attention state" + "notes": "MiniMax M3 MoE with three leading dense blocks and sparse-attention state", + "class": "causal_generation" } }, { @@ -3781,7 +3876,8 @@ "cache_policy": "immutable-local", "estimated_model_bytes": 2594836480 }, - "notes": "Spark X2.5 dense mixed-window attention architecture coverage" + "notes": "Spark X2.5 dense mixed-window attention architecture coverage", + "class": "causal_generation" } }, { @@ -3999,6 +4095,333 @@ ], "notes": "SafeTensors direct-load fixture used across every supported load-time quantization." }, + { + "id": "family-nomic-bert-embedding", + "family": "nomic-bert-embedding", + "suites": [ + "llama-family-certification" + ], + "cadences": [ + "llama-bump", + "manual-full", + "nightly" + ], + "capability_tags": [ + "gguf", + "embedding" + ], + "artifact": { + "repo": "nomic-ai/nomic-embed-text-v1.5-GGUF", + "revision": "0188c9bf409793f810680a5a431e7b899c46104c", + "selector": "Q8_0", + "files": [ + { + "path": "nomic-embed-text-v1.5.Q8_0.gguf", + "size_bytes": 146146432, + "sha256": "3e24342164b3d94991ba9692fdc0dd08e3fd7362e0aacc396a9a5c54a544c3b7" + } + ] + }, + "certification": { + "class": "embedding", + "profile": "workload-oracle", + "execution": { + "trunk_layers": 12, + "mtp_layers": 0, + "activation_width": 768, + "speculative_policy": "disabled" + }, + "resources": { + "runner_role": "family-certify", + "cache_policy": "immutable-local", + "estimated_model_bytes": 145389792 + }, + "notes": "Local full-model embedding, HTTP and official SDK smoke, and pinned CPU monolithic batch/single-vector parity; staged tensor filtering is intentionally unsupported", + "evidence": { + "fixture": "scripts/workload_fixtures.py#EMBEDDING_INPUTS", + "comparison": "batch and individual vectors within 1e-4 coordinate error and 0.99999 cosine" + } + } + }, + { + "id": "family-jina-bert-v2-rerank", + "family": "jina-bert-v2-rerank", + "suites": [ + "llama-family-certification" + ], + "cadences": [ + "llama-bump", + "manual-full", + "nightly" + ], + "capability_tags": [ + "gguf", + "rerank" + ], + "artifact": { + "repo": "ggml-org/jina-reranker-v1-turbo-en-GGUF", + "revision": "607d8664c787e517e5d6e339d21f680f9002c931", + "selector": "F16", + "files": [ + { + "path": "Jina-Bert-Implementation-38M-F16.gguf", + "size_bytes": 76971168, + "sha256": "71abc010bb3dce97812ee971509a5cb6ff6f6b8cfffd8480129242f605521fca" + } + ] + }, + "certification": { + "class": "rerank", + "profile": "workload-oracle", + "execution": { + "trunk_layers": 6, + "mtp_layers": 0, + "activation_width": 384, + "speculative_policy": "disabled" + }, + "resources": { + "runner_role": "family-certify", + "cache_policy": "immutable-local", + "estimated_model_bytes": 75293188 + }, + "notes": "Local full-model cross-encoder rerank and HTTP smoke with pinned CPU monolithic score/order parity; staged tensor filtering is intentionally unsupported", + "evidence": { + "fixture": "scripts/workload_fixtures.py#RERANK_DOCUMENTS", + "comparison": "scores within 1e-4 and identical document order" + } + } + }, + { + "id": "family-t5-encoder-decoder", + "family": "t5-encoder-decoder", + "suites": [ + "llama-family-certification" + ], + "cadences": [ + "llama-bump", + "manual-full", + "nightly" + ], + "capability_tags": [ + "gguf", + "encoder_decoder" + ], + "artifact": { + "repo": "Felladrin/gguf-flan-t5-small", + "revision": "d71c51f67519edd3154527c2d8f20288bdde9705", + "selector": "Q8_0", + "files": [ + { + "path": "flan-t5-small.Q8_0.gguf", + "size_bytes": 113709824, + "sha256": "f7f769c360b1ba830b10dd3b7e7d146dbcc4d487962be7dd806d7d52e0a9c2f0" + } + ] + }, + "certification": { + "class": "encoder_decoder", + "profile": "workload-oracle", + "execution": { + "trunk_layers": 8, + "mtp_layers": 0, + "activation_width": 512, + "speculative_policy": "disabled" + }, + "resources": { + "runner_role": "family-certify", + "cache_policy": "immutable-local", + "estimated_model_bytes": 112678400 + }, + "notes": "Local full-model encoder-decoder generation and HTTP smoke with pinned CPU monolithic completion text parity; cross-attention staging is intentionally unsupported", + "evidence": { + "fixture": "scripts/workload_fixtures.py#ENCODER_DECODER_PROMPT", + "comparison": "identical normalized greedy text from pinned llama-completion" + } + } + }, + { + "id": "family-paddleocr", + "family": "paddleocr", + "suites": [ + "llama-family-certification" + ], + "cadences": [ + "llama-bump", + "manual-full", + "nightly" + ], + "capability_tags": [ + "gguf", + "ocr" + ], + "artifact": { + "repo": "PaddlePaddle/PaddleOCR-VL-1.6-GGUF", + "revision": "511b09642bb324401f15f97cc23bc67e8f0a291d", + "selector": "BF16", + "files": [ + { + "path": "PaddleOCR-VL-1.6-GGUF.gguf", + "size_bytes": 935769056, + "sha256": "f3ae46ec885050acf4b3d31944431e1fd90d50664fb09126af4a3c050ba14ee8" + } + ] + }, + "certification": { + "class": "ocr", + "profile": "workload-oracle", + "execution": { + "trunk_layers": 18, + "mtp_layers": 0, + "activation_width": 1024, + "speculative_policy": "disabled" + }, + "resources": { + "runner_role": "family-certify", + "cache_policy": "immutable-local", + "estimated_model_bytes": 933384192, + "startup_timeout_secs": 900 + }, + "notes": "Local full-model OCR and HTTP smoke with the pinned PaddleOCR projector and mandatory generated-image monolithic parity; projector and trunk remain colocated", + "evidence": { + "fixture": "scripts/generate-ocr-oracle-fixture.py#MESH 42", + "comparison": "normalized monolithic text parity plus independent MESH 42 label" + }, + "mmproj_artifact": { + "repo": "PaddlePaddle/PaddleOCR-VL-1.6-GGUF", + "revision": "511b09642bb324401f15f97cc23bc67e8f0a291d", + "selector": "BF16", + "files": [ + { + "path": "PaddleOCR-VL-1.6-GGUF-mmproj.gguf", + "size_bytes": 881770560, + "sha256": "204d757d7610d9b3faab10d506d69e5b244e32bf765e2bab2d0167e65e0a058a" + } + ] + } + } + }, + { + "id": "family-qwen3tts", + "family": "qwen3tts", + "suites": [ + "llama-family-certification" + ], + "cadences": [ + "llama-bump", + "manual-full", + "nightly" + ], + "capability_tags": [ + "gguf", + "speech_synthesis" + ], + "artifact": { + "repo": "ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF", + "revision": "ca27d74bc954b73dadab5b71ca265d87fc861a7c", + "selector": "Q8_0", + "files": [ + { + "path": "Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf", + "size_bytes": 1847874400, + "sha256": "ac7931aeb2e7aad1a6ed6602d353a5679c9d096b18ce8204ac730a8408d572e1" + } + ] + }, + "certification": { + "class": "speech_synthesis", + "profile": "workload-oracle", + "execution": { + "trunk_layers": 28, + "mtp_layers": 0, + "activation_width": 2048, + "speculative_policy": "disabled" + }, + "resources": { + "runner_role": "family-certify", + "cache_policy": "immutable-local", + "estimated_model_bytes": 1841844224, + "startup_timeout_secs": 600 + }, + "notes": "Local full-model speech-synthesis and HTTP WAV smoke with deterministic pinned monolithic PCM parity; distributed staging is intentionally unsupported", + "evidence": { + "fixture": "scripts/skippy-tts-oracle.py#PROMPT", + "comparison": "fixed-seed PCM format and length; RMS error at most 2 percent and waveform cosine at least 0.9995" + }, + "mmproj_artifact": { + "repo": "ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF", + "revision": "ca27d74bc954b73dadab5b71ca265d87fc861a7c", + "selector": "Q8_0", + "files": [ + { + "path": "mmproj-Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf", + "size_bytes": 446422912, + "sha256": "6fd65188839bcd6ecc91b277ad471e22a0edfada4699a0fe82f1165c18cfcce2" + } + ] + } + } + }, + { + "id": "family-ultravox", + "family": "ultravox", + "suites": [ + "llama-family-certification" + ], + "cadences": [ + "llama-bump", + "manual-full", + "nightly" + ], + "capability_tags": [ + "gguf", + "speech_recognition" + ], + "artifact": { + "repo": "ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF", + "revision": "5390c7c41cbd6f261f7f205fc0c5ae61bbdca650", + "selector": "Q8_0", + "files": [ + { + "path": "Llama-3.2-1B-Instruct-Q8_0.gguf", + "size_bytes": 1321083008, + "sha256": "432f310a77f4650a88d0fd59ecdd7cebed8d684bafea53cbff0473542964f0c3" + } + ] + }, + "certification": { + "class": "speech_recognition", + "profile": "workload-oracle", + "execution": { + "trunk_layers": 16, + "mtp_layers": 0, + "activation_width": 2048, + "speculative_policy": "disabled" + }, + "resources": { + "runner_role": "family-certify", + "cache_policy": "immutable-local", + "estimated_model_bytes": 1313251456, + "startup_timeout_secs": 600 + }, + "notes": "Local full-model audio-to-text and HTTP smoke with the pinned Ultravox projector and monolithic transcript parity; unlabeled WAV does not establish transcription accuracy", + "evidence": { + "fixture": "ci/llama-canary/fixtures/audio-smoke.wav", + "comparison": "identical normalized transcript against pinned llama-server; fixture is unlabeled" + }, + "mmproj_artifact": { + "repo": "ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF", + "revision": "5390c7c41cbd6f261f7f205fc0c5ae61bbdca650", + "selector": "F16", + "files": [ + { + "path": "mmproj-ultravox-v0_5-llama-3_2-1b-f16.gguf", + "size_bytes": 1371123616, + "sha256": "b34dde1835752949d6b960528269af93c92fec91c61ea0534fcc73f96c1ed8b2" + } + ] + } + } + }, { "id": "family-inkling", "family": "inkling", @@ -4051,6 +4474,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 42, @@ -4112,6 +4536,7 @@ ] }, "certification": { + "class": "causal_generation", "profile": "full", "execution": { "trunk_layers": 48, diff --git a/crates/mesh-llm-host-runtime/src/api/routes/logs/events/query.rs b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/query.rs index a584b9f0f0..401ef3d66e 100644 --- a/crates/mesh-llm-host-runtime/src/api/routes/logs/events/query.rs +++ b/crates/mesh-llm-host-runtime/src/api/routes/logs/events/query.rs @@ -291,17 +291,14 @@ fn parse_query(path: &str) -> Result { )); } } - "severity" => { - if audit_mode { - let severity = parse_audit_severity(&value)?; - if audit_selection.severity.is_some() { - return Err(LogsError::InvalidQuery("duplicate audit severity")); - } - audit_selection.severity = Some(severity); - } else { - return Err(LogsError::InvalidQuery("unknown event stream parameter")); + "severity" if audit_mode => { + let severity = parse_audit_severity(&value)?; + if audit_selection.severity.is_some() { + return Err(LogsError::InvalidQuery("duplicate audit severity")); } + audit_selection.severity = Some(severity); } + "severity" => return Err(LogsError::InvalidQuery("unknown event stream parameter")), "cursor" if cursor.is_none() && !audit_mode => { cursor = Some(Cursor::parse(nonempty(&value)?)?); } diff --git a/crates/mesh-llm-host-runtime/src/inference/skippy/mod.rs b/crates/mesh-llm-host-runtime/src/inference/skippy/mod.rs index d2193a555a..33564d05c2 100644 --- a/crates/mesh-llm-host-runtime/src/inference/skippy/mod.rs +++ b/crates/mesh-llm-host-runtime/src/inference/skippy/mod.rs @@ -30,11 +30,12 @@ use std::{ use anyhow::{Context, Result}; use async_trait::async_trait; use openai_frontend::{ + AudioResponse, AudioSpeechRequest, AudioTranscriptionRequest, AudioTranscriptionResponse, ChatCompletionRequest, ChatCompletionResponse, ChatCompletionStream, CompactingOpenAiBackend, - CompactionConfig, CompletionRequest, CompletionResponse, CompletionStream, - GuardedOpenAiBackend, GuardrailMode, GuardrailPolicy, GuardrailPolicyHandle, + CompactionConfig, CompletionRequest, CompletionResponse, CompletionStream, EmbeddingResponse, + EmbeddingsRequest, GuardedOpenAiBackend, GuardrailMode, GuardrailPolicy, GuardrailPolicyHandle, GuardrailTelemetrySink, ModelObject, OpenAiBackend, OpenAiHookPolicy, OpenAiRequestContext, - OpenAiResult, + OpenAiResult, RerankRequest, RerankResponse, }; use skippy_protocol::{FlashAttentionType, LoadMode, StageConfig, StageDevice, StageKvCacheConfig}; use skippy_runtime::{ModelInfo, MtpSource}; @@ -659,6 +660,27 @@ impl SkippyModelHandle { self.runtime.output_activation_boundary() } + /// Classify the loaded native runtime, including speech-capable projectors. + pub(crate) fn workload_class(&self) -> Result { + if self.runtime.supports_speech_synthesis() { + return Ok(crate::mesh::ModelWorkloadClass::SpeechSynthesis); + } + let workload = self + .runtime + .workload_info() + .context("read loaded model workload contract")?; + Ok(match workload.kind { + skippy_runtime::ModelWorkload::CausalGeneration => { + crate::mesh::ModelWorkloadClass::CausalGeneration + } + skippy_runtime::ModelWorkload::Embedding => crate::mesh::ModelWorkloadClass::Embedding, + skippy_runtime::ModelWorkload::Rerank => crate::mesh::ModelWorkloadClass::Rerank, + skippy_runtime::ModelWorkload::EncoderDecoder => { + crate::mesh::ModelWorkloadClass::EncoderDecoder + } + }) + } + fn resolved_mtp_source( native_mtp_enabled: bool, native_mtp_draft_model_path: Option<&Path>, @@ -1228,6 +1250,51 @@ impl OpenAiBackend for SkippyModelHandle { ) -> OpenAiResult { self.backend.completion_stream(request, context).await } + + /// Forward embeddings and request context without chat processing. + async fn embeddings( + &self, + request: EmbeddingsRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.embeddings(request, context).await + } + + /// Forward reranking and request context without chat processing. + async fn rerank( + &self, + request: RerankRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.rerank(request, context).await + } + + /// Forward speech generation and request context unchanged. + async fn audio_speech( + &self, + request: AudioSpeechRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.audio_speech(request, context).await + } + + /// Forward multipart transcription and request context unchanged. + async fn audio_transcription( + &self, + request: AudioTranscriptionRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.audio_transcription(request, context).await + } + + /// Forward multipart translation and request context unchanged. + async fn audio_translation( + &self, + request: AudioTranscriptionRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.audio_translation(request, context).await + } } pub(crate) fn single_stage_config(options: &SkippyModelLoadOptions) -> Result { diff --git a/crates/mesh-llm-host-runtime/src/inference/skippy/split-certified.json b/crates/mesh-llm-host-runtime/src/inference/skippy/split-certified.json index 8fde004e09..b9984d235d 100644 --- a/crates/mesh-llm-host-runtime/src/inference/skippy/split-certified.json +++ b/crates/mesh-llm-host-runtime/src/inference/skippy/split-certified.json @@ -2,8 +2,8 @@ "schema_version": 1, "native_recipe": { "llama_upstream_sha": "661643e43079a4ee6faab4c1895291767b67ea8d", - "skippy_abi": "0.1.54", - "patch_queue_sha256": "3b12d21a496aa526288b8106c32a6957f5c924f981e2ee708fcd4d32d7ba02c9" + "skippy_abi": "0.1.55", + "patch_queue_sha256": "267df9ed1c48e82b3423041552ecf5a51c481237f6de87037c24bc3ae04af038" }, "models": [ { diff --git a/crates/mesh-llm-host-runtime/src/logging/openai_lifecycle.rs b/crates/mesh-llm-host-runtime/src/logging/openai_lifecycle.rs index b199bc20b1..75e899f4bc 100644 --- a/crates/mesh-llm-host-runtime/src/logging/openai_lifecycle.rs +++ b/crates/mesh-llm-host-runtime/src/logging/openai_lifecycle.rs @@ -780,9 +780,15 @@ fn terminal_outcome(result: OpenAiTerminalResult) -> TerminalOutcome { } } +/// Map backend operations to bounded labels, separating non-chat and streaming workloads. const fn operation_label(operation: OpenAiBackendOperation) -> &'static str { match operation { OpenAiBackendOperation::Models => "models", + OpenAiBackendOperation::Embeddings => "embeddings", + OpenAiBackendOperation::Rerank => "rerank", + OpenAiBackendOperation::AudioSpeech => "audio_speech", + OpenAiBackendOperation::AudioTranscription => "audio_transcription", + OpenAiBackendOperation::AudioTranslation => "audio_translation", OpenAiBackendOperation::ChatCompletion => "chat_completion", OpenAiBackendOperation::ChatCompletionStream => "chat_completion_stream", OpenAiBackendOperation::Completion => "completion", diff --git a/crates/mesh-llm-host-runtime/src/logging/request_metadata.rs b/crates/mesh-llm-host-runtime/src/logging/request_metadata.rs index 0cdb32542b..aa1b07e0a3 100644 --- a/crates/mesh-llm-host-runtime/src/logging/request_metadata.rs +++ b/crates/mesh-llm-host-runtime/src/logging/request_metadata.rs @@ -158,12 +158,18 @@ impl RequestSummaryMetadata { } } +/// Project known endpoints to fixed log labels; unknown paths contribute no label. const fn openai_route_label(route: OpenAiFrontendRoute) -> Option<&'static str> { match route { OpenAiFrontendRoute::Health => Some("health"), OpenAiFrontendRoute::Healthz => Some("healthz"), OpenAiFrontendRoute::Readyz => Some("readyz"), OpenAiFrontendRoute::Models => Some("models"), + OpenAiFrontendRoute::Embeddings => Some("embeddings"), + OpenAiFrontendRoute::Rerank => Some("rerank"), + OpenAiFrontendRoute::AudioSpeech => Some("audio_speech"), + OpenAiFrontendRoute::AudioTranscriptions => Some("audio_transcriptions"), + OpenAiFrontendRoute::AudioTranslations => Some("audio_translations"), OpenAiFrontendRoute::ChatCompletions => Some("chat_completions"), OpenAiFrontendRoute::Completions => Some("completions"), OpenAiFrontendRoute::Responses => Some("responses"), diff --git a/crates/mesh-llm-host-runtime/src/mesh/identity_persistence.rs b/crates/mesh-llm-host-runtime/src/mesh/identity_persistence.rs index 5bb607462d..52e1ec83fa 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/identity_persistence.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/identity_persistence.rs @@ -249,7 +249,7 @@ pub(crate) async fn load_or_create_key() -> Result { pub fn default_node_key_path() -> Result { #[cfg(test)] if std::env::var_os("MESH_LLM_TEST_HOME").is_some() - && !std::env::var_os("MESH_LLM_NODE_KEY_PATH").is_some_and(|path| !path.is_empty()) + && std::env::var_os("MESH_LLM_NODE_KEY_PATH").is_none_or(|path| path.is_empty()) { return Ok(identity_home_dir().join(".mesh-llm").join("key")); } diff --git a/crates/mesh-llm-host-runtime/src/mesh/mod.rs b/crates/mesh-llm-host-runtime/src/mesh/mod.rs index 7f26f98679..c3158a9d81 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/mod.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/mod.rs @@ -6,9 +6,10 @@ //! latency-sensitive `skippy-stage/2` ALPN. pub use mesh_llm_types::mesh::{ - ModelDemand, ModelRuntimeDescriptor, ModelSourceKind, ServedModelDescriptor, - ServedModelIdentity, ServedModelMetadata, infer_available_model_descriptors, - infer_local_served_model_descriptor, infer_served_model_descriptors, + ModelDemand, ModelRuntimeDescriptor, ModelSourceKind, ModelWorkloadClass, + ServedModelDescriptor, ServedModelIdentity, ServedModelMetadata, + infer_available_model_descriptors, infer_local_served_model_descriptor, + infer_served_model_descriptors, }; use anyhow::{Context, Result}; @@ -90,7 +91,7 @@ mod heartbeat; mod host_role_claims; mod identity_persistence; mod lan_bootstrap; -mod model_identity; +pub(crate) mod model_identity; mod node; mod node_identity; mod node_requirements; diff --git a/crates/mesh-llm-host-runtime/src/mesh/model_identity.rs b/crates/mesh-llm-host-runtime/src/mesh/model_identity.rs index 58f5a80e57..1105ee362e 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/model_identity.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/model_identity.rs @@ -1,3 +1,5 @@ +//! Public routing names derived from served identity and the loaded catalog. + use super::*; pub(crate) fn infer_remote_served_descriptors( @@ -216,6 +218,59 @@ pub(crate) fn identity_hash_for(input: &str) -> String { hex::encode(hasher.finalize()) } +/// Expose only remotely resolvable model identities as public catalog IDs. +pub(crate) fn public_model_id_from_identity(identity: &ServedModelIdentity) -> Option { + match identity.source_kind { + ModelSourceKind::HuggingFace => identity + .repository + .as_deref() + .map(|repo| { + let selector = identity + .artifact + .as_deref() + .and_then(model_ref::quant_selector_from_gguf_file) + .or_else(|| identity.artifact.clone()); + model_ref::format_model_ref(repo, identity.revision.as_deref(), selector.as_deref()) + }) + .or_else(|| { + identity + .canonical_ref + .as_deref() + .and_then(|model_ref| model_ref::ModelRef::parse(model_ref).ok()) + .map(|model_ref| model_ref.display_id()) + }), + ModelSourceKind::Catalog => identity + .canonical_ref + .as_deref() + .and_then(|model_ref| model_ref::ModelRef::parse(model_ref).ok()) + .map(|model_ref| model_ref.display_id()), + ModelSourceKind::LocalGguf | ModelSourceKind::DirectUrl | ModelSourceKind::Unknown => None, + } +} + +/// Normalize demand references without converting local paths into remote model identities. +pub(crate) fn canonical_demand_model_ref(model: &str) -> String { + if let Ok(model_ref) = model_ref::ModelRef::parse(model) { + return model_ref.display_id(); + } + crate::models::find_loaded_remote_catalog_model_exact(model) + .map(|remote_model| crate::models::remote_catalog_model_ref(&remote_model)) + .unwrap_or_else(|| model.to_string()) +} + +/// Match exactly the same public alias that peer HTTP discovery advertises. +/// Do not infer identity from similar basenames or borrow another peer's facts. +pub(crate) fn descriptor_matches_routable_name( + descriptor: &ServedModelDescriptor, + name: &str, +) -> bool { + let identity = &descriptor.identity; + identity.model_name == name + || public_model_id_from_identity(identity) + .unwrap_or_else(|| canonical_demand_model_ref(&identity.model_name)) + == name +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/mesh-llm-host-runtime/src/mesh/node.rs b/crates/mesh-llm-host-runtime/src/mesh/node.rs index 7aef15d97d..f46cb74ce9 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/node.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/node.rs @@ -1448,7 +1448,7 @@ impl Node { }) } - #[cfg(test)] + /// Snapshot only this node's models for endpoint-local workload admission. pub async fn served_model_descriptors(&self) -> Vec { self.served_model_descriptors.lock().await.clone() } diff --git a/crates/mesh-llm-host-runtime/src/mesh/peer_state.rs b/crates/mesh-llm-host-runtime/src/mesh/peer_state.rs index a07b27caa0..e4d39f5c69 100644 --- a/crates/mesh-llm-host-runtime/src/mesh/peer_state.rs +++ b/crates/mesh-llm-host-runtime/src/mesh/peer_state.rs @@ -492,44 +492,6 @@ impl PeerInfo { } } -pub(crate) fn public_model_id_from_identity(identity: &ServedModelIdentity) -> Option { - match identity.source_kind { - ModelSourceKind::HuggingFace => identity - .repository - .as_deref() - .map(|repo| { - let selector = identity - .artifact - .as_deref() - .and_then(model_ref::quant_selector_from_gguf_file) - .or_else(|| identity.artifact.clone()); - model_ref::format_model_ref(repo, identity.revision.as_deref(), selector.as_deref()) - }) - .or_else(|| { - identity - .canonical_ref - .as_deref() - .and_then(|model_ref| model_ref::ModelRef::parse(model_ref).ok()) - .map(|model_ref| model_ref.display_id()) - }), - ModelSourceKind::Catalog => identity - .canonical_ref - .as_deref() - .and_then(|model_ref| model_ref::ModelRef::parse(model_ref).ok()) - .map(|model_ref| model_ref.display_id()), - ModelSourceKind::LocalGguf | ModelSourceKind::DirectUrl | ModelSourceKind::Unknown => None, - } -} - -pub(crate) fn canonical_demand_model_ref(model: &str) -> String { - if let Ok(model_ref) = model_ref::ModelRef::parse(model) { - return model_ref.display_id(); - } - crate::models::find_loaded_remote_catalog_model_exact(model) - .map(|remote_model| crate::models::remote_catalog_model_ref(&remote_model)) - .unwrap_or_else(|| model.to_string()) -} - /// Peers not directly verified within this window are considered stale /// and excluded from gossip propagation. After 2x this duration they're removed entirely. pub(crate) const PEER_STALE_SECS: u64 = 180; // 3 minutes diff --git a/crates/mesh-llm-host-runtime/src/models/profile.rs b/crates/mesh-llm-host-runtime/src/models/profile.rs index 6ed3c90783..167feca0be 100644 --- a/crates/mesh-llm-host-runtime/src/models/profile.rs +++ b/crates/mesh-llm-host-runtime/src/models/profile.rs @@ -34,6 +34,7 @@ pub(crate) fn served_model_metadata_for_path( let parameter_count_b = parameter_count.map(|total| total as f64 / 1e9); let kv_head_count = meta.effective_kv_head_count(); crate::mesh::ServedModelMetadata { + workload_class: None, architecture: non_empty(meta.architecture), parameter_size, parameter_count_b, diff --git a/crates/mesh-llm-host-runtime/src/network/openai/auto_route.rs b/crates/mesh-llm-host-runtime/src/network/openai/auto_route.rs index 83d5373b21..2dbd3a1b36 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/auto_route.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/auto_route.rs @@ -6,6 +6,7 @@ //! should use that model instead of spending an agent turn on a target we just //! proved unhealthy or too small. +use super::workload_routing; use crate::inference::election; use crate::mesh; use crate::network::affinity::AffinityRouter; @@ -53,25 +54,31 @@ async fn context_compatible_targets( compatible } +/// Require a workload-compatible target that fits context and passes strict health routing. pub(crate) async fn model_has_eligible_target( node: &mesh::Node, model: &str, required_tokens: Option, + request_path: &str, candidates: &[election::InferenceTarget], affinity: &AffinityRouter, ) -> bool { + let candidates = + workload_routing::eligible_targets(node, model, request_path, candidates).await; let context_compatible = - context_compatible_targets(node, model, required_tokens, candidates).await; + context_compatible_targets(node, model, required_tokens, &candidates).await; if !has_routable_candidate(&context_compatible) { return false; } has_routable_candidate(&affinity.route_strict_eligible_candidates(model, &context_compatible)) } +/// Apply the same workload, context, and health admission to advertised remote hosts. pub(crate) async fn model_has_eligible_remote_host( node: &mesh::Node, model: &str, required_tokens: Option, + request_path: &str, affinity: &AffinityRouter, ) -> bool { let targets: Vec = node @@ -80,7 +87,15 @@ pub(crate) async fn model_has_eligible_remote_host( .into_iter() .map(election::InferenceTarget::Remote) .collect(); - model_has_eligible_target(node, model, required_tokens, &targets, affinity).await + model_has_eligible_target( + node, + model, + required_tokens, + request_path, + &targets, + affinity, + ) + .await } /// Whether this node itself serves `model`, independent of the target table. @@ -111,15 +126,25 @@ pub(crate) fn pool_for_ready_models<'a>( } } +/// Select model names with a ready remote host for this endpoint and context budget. pub(crate) async fn ready_remote_models<'a>( node: &mesh::Node, required_tokens: Option, + request_path: &str, available: &[router::RoutingCandidate<'a>], affinity: &AffinityRouter, ) -> Vec<&'a str> { let mut ready_models = Vec::new(); for candidate in available { - if model_has_eligible_remote_host(node, candidate.name, required_tokens, affinity).await { + if model_has_eligible_remote_host( + node, + candidate.name, + required_tokens, + request_path, + affinity, + ) + .await + { ready_models.push(candidate.name); } } diff --git a/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs b/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs index 75508b3d37..af6d0b7a9e 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs @@ -7,6 +7,9 @@ use crate::network::openai::auto_route; use crate::network::openai::automatic; use crate::network::openai::client_stream::ClientStream; use crate::network::openai::transport as proxy; +use crate::network::openai::workload_routing::{ + self, is_audio_upload_path, model_satisfies_request_workload, request_workload_class, +}; use crate::network::router; use crate::plugin::openai_exchange::{ ClientNonceSource, OpenAiExchangeChannel, OpenAiExchangeDispatchPath, OpenAiExchangeEnvelope, @@ -52,6 +55,13 @@ enum AutoRouteResolution { classification: Option, }, MediaUnsupported, + WorkloadUnsupported(mesh::ModelWorkloadClass), +} + +#[derive(Debug)] +enum AutoRouteRejection { + MediaUnsupported, + WorkloadUnsupported(mesh::ModelWorkloadClass), } struct IngressRouteContext<'a> { @@ -243,6 +253,7 @@ async fn collect_available_models_for_auto_route( available_models } +/// Admit explicit workloads or select a compatible automatic route, preserving committee mode. async fn resolve_auto_routed_model( node: &mesh::Node, request: &mut proxy::BufferedHttpRequest, @@ -252,6 +263,7 @@ async fn resolve_auto_routed_model( required_tokens: Option, affinity: &affinity::AffinityRouter, ) -> AutoRouteResolution { + let requested_workload = request_workload_class(&request.client_path); // An explicitly named model routes to itself. The automatic directive (in // either spelling) resolves here, so `mesh` reaches the same // media-capability filter and readiness/affinity selection as `auto` — @@ -260,6 +272,11 @@ async fn resolve_auto_routed_model( if let Some(model) = request.model_name.as_deref() && !automatic::is_directive(model) { + if let Some(workload) = requested_workload + && !model_satisfies_request_workload(model, workload, &request.client_path, descriptors) + { + return AutoRouteResolution::WorkloadUnsupported(workload); + } return AutoRouteResolution::Continue { effective_model: request.model_name.clone(), classification: None, @@ -267,22 +284,28 @@ async fn resolve_auto_routed_model( } request.ensure_body_json(); - let Some(body_json) = request.body_json.as_ref() else { + let body_json = request.body_json.as_ref(); + if body_json.is_none() && requested_workload.is_none() { return AutoRouteResolution::Continue { effective_model: None, classification: None, }; - }; + } automatic::warn_if_deprecated_alias(request.model_name.as_deref()); - let mode = automatic::serving_mode(automatic::AutomaticRequest { - model: request.model_name.as_deref(), - // The forwarded path: `/v1/responses` has already been normalised onto - // chat completions by this point, so it stays committee-eligible. - path: &request.path, - body: body_json, - }); + let mode = body_json.map_or( + automatic::ServingMode::SingleModel(automatic::SingleModelReason::NonChatRequest), + |body| { + automatic::serving_mode(automatic::AutomaticRequest { + model: request.model_name.as_deref(), + // The forwarded path: `/v1/responses` has already been normalised onto + // chat completions by this point, so it stays committee-eligible. + path: &request.path, + body, + }) + }, + ); match mode { // Committee mode keeps the directive as the effective model so the MoA // gateway picks the request up. @@ -300,36 +323,41 @@ async fn resolve_auto_routed_model( ), } - let classification = router::classify(body_json); - let media = router::media_requirements(body_json); + let classification = body_json.map_or_else( + || router::Classification { + category: router::Category::Chat, + complexity: router::Complexity::Quick, + needs_tools: false, + has_media_inputs: is_audio_upload_path(&request.client_path), + }, + router::classify, + ); + let media = workload_routing::request_media(&request.client_path, body_json); let available_models = collect_available_models_for_auto_route(node, targets, plugin_manager).await; - let metrics = node.routing_metrics(); - let available: Vec> = available_models - .iter() - .map(|name| { - let caps = proxy::capabilities_for_model(name, descriptors); - let (tps_hint, throughput_samples) = metrics - .tps_for_model(name) - .map(|(t, s)| (Some(t), s)) - .unwrap_or((None, 0)); - router::RoutingCandidate { - name: name.as_str(), - caps, - parameter_count_b: proxy::descriptor_metadata_for_model(name, descriptors) - .and_then(|metadata| metadata.parameter_count_b), - tps_hint, - throughput_samples, - } - }) - .collect(); + let available = workload_routing::routing_candidates( + node, + &available_models, + &request.client_path, + descriptors, + ); + if available.is_empty() + && let Some(workload) = requested_workload + { + return AutoRouteResolution::WorkloadUnsupported(workload); + } let Some(available) = router::filter_media_compatible_candidates(&available, &media) else { - proxy::release_request_objects(node, &request.request_object_request_ids).await; return AutoRouteResolution::MediaUnsupported; }; - let available = - auto_route_pool_for_ready_models(node, targets, required_tokens, &available, affinity) - .await; + let available = auto_route_pool_for_ready_models( + node, + targets, + required_tokens, + &request.client_path, + &available, + affinity, + ) + .await; let effective_model = router::pick_model_classified(&classification, &available).map(|name| { tracing::info!( @@ -347,10 +375,12 @@ async fn resolve_auto_routed_model( } } +/// Prefer ready ingress models, retaining the admitted pool if none is ready yet. async fn auto_route_pool_for_ready_models<'a>( node: &mesh::Node, targets: &election::ModelTargets, required_tokens: Option, + request_path: &str, available: &[router::RoutingCandidate<'a>], affinity: &affinity::AffinityRouter, ) -> Vec> { @@ -361,6 +391,7 @@ async fn auto_route_pool_for_ready_models<'a>( targets, candidate.name, required_tokens, + request_path, affinity, ) .await @@ -371,19 +402,23 @@ async fn auto_route_pool_for_ready_models<'a>( auto_route::pool_for_ready_models(available, &ready_models) } +/// Check workload-aware ingress readiness, including remote and local-startup fallbacks. async fn auto_route_model_has_ready_ingress_target( node: &mesh::Node, targets: &election::ModelTargets, model: &str, required_tokens: Option, + request_path: &str, affinity: &affinity::AffinityRouter, ) -> bool { - let local_candidates = targets.candidates(model); + let local_candidates = + workload_routing::ingress_candidates(node, model, request_path, targets).await; if contains_routable_candidate(&local_candidates) { return auto_route::model_has_eligible_target( node, model, required_tokens, + request_path, &local_candidates, affinity, ) @@ -401,6 +436,7 @@ async fn auto_route_model_has_ready_ingress_target( node, model, required_tokens, + request_path, &remote_candidates, affinity, ) @@ -888,11 +924,12 @@ fn prepare_cache_routing_body( } } +/// Resolve workload admission before enabling JSON hooks and recording model demand. async fn prepare_auto_route_decision( request: &mut proxy::BufferedHttpRequest, ctx: &IngressRouteContext<'_>, descriptors: &[crate::mesh::ServedModelDescriptor], -) -> Result { +) -> Result { let required_tokens = proxy::request_context_budget(request); match resolve_auto_routed_model( ctx.node, @@ -919,10 +956,31 @@ async fn prepare_auto_route_decision( required_tokens, }) } - AutoRouteResolution::MediaUnsupported => Err(()), + AutoRouteResolution::MediaUnsupported => Err(AutoRouteRejection::MediaUnsupported), + AutoRouteResolution::WorkloadUnsupported(workload) => { + Err(AutoRouteRejection::WorkloadUnsupported(workload)) + } } } +/// Return a path-specific unsupported-workload response and record the rejection. +async fn send_workload_unsupported( + tcp_stream: ClientStream, + workload: mesh::ModelWorkloadClass, + path: &str, + route_observer: OpenAiRouteObserver<'_>, +) -> proxy::RouteDispatchOutcome { + let message = if is_audio_upload_path(path) { + "no served model advertises support for this audio-to-text endpoint".to_string() + } else { + format!("no served model advertises the required {workload:?} workload") + }; + response_outcome( + 422, + proxy::send_error_observed(tcp_stream, 422, &message, route_observer).await, + ) +} + async fn send_media_unsupported( tcp_stream: ClientStream, route_observer: OpenAiRouteObserver<'_>, @@ -939,6 +997,26 @@ async fn send_media_unsupported( ) } +/// Release request-scoped media objects before reporting an automatic-routing rejection. +async fn send_auto_route_rejection( + tcp_stream: ClientStream, + rejection: AutoRouteRejection, + node: &mesh::Node, + request_object_request_ids: &[String], + path: &str, + route_observer: OpenAiRouteObserver<'_>, +) -> proxy::RouteDispatchOutcome { + proxy::release_request_objects(node, request_object_request_ids).await; + match rejection { + AutoRouteRejection::MediaUnsupported => { + send_media_unsupported(tcp_stream, route_observer).await + } + AutoRouteRejection::WorkloadUnsupported(workload) => { + send_workload_unsupported(tcp_stream, workload, path, route_observer).await + } + } +} + fn callable_models_with_local_served( targets: &election::ModelTargets, local_models: Vec, @@ -1168,8 +1246,16 @@ async fn handle_buffered_api_request( let decision = match prepare_auto_route_decision(&mut request, &ctx.route, &descriptors).await { Ok(decision) => decision, - Err(()) => { - let outcome = send_media_unsupported(tcp_stream, lifecycle.route_observer()).await; + Err(rejection) => { + let outcome = send_auto_route_rejection( + tcp_stream, + rejection, + ctx.route.node, + &request.request_object_request_ids, + &request.client_path, + lifecycle.route_observer(), + ) + .await; lifecycle.terminal(terminal_outcome_for_dispatch(outcome)); return; } @@ -1437,6 +1523,14 @@ mod durable_artifacts; #[path = "ingress_tests/automatic_routing.rs"] mod automatic_routing; +#[cfg(test)] +#[path = "ingress_tests/audio_workloads.rs"] +mod audio_workloads; + +#[cfg(test)] +#[path = "ingress_tests/request_object_cleanup.rs"] +mod request_object_cleanup; + #[cfg(test)] #[path = "ingress_tests/tests.rs"] mod tests; diff --git a/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/audio_workloads.rs b/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/audio_workloads.rs new file mode 100644 index 0000000000..f5dc1811c9 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/audio_workloads.rs @@ -0,0 +1,120 @@ +use super::*; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; + +const MODEL: &str = "audio-workload"; + +/// Build an audio upload whose file bytes cannot safely pass through JSON hooks. +fn multipart(model: &str) -> Vec { + [b"--test\r\nContent-Disposition: form-data; name=\"file\"; filename=\"audio.wav\"\r\nContent-Type: audio/wav\r\n\r\nRIFF\0{\xff}\x80\r\n--test\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\n".as_slice(), model.as_bytes(), b"\r\n--test--\r\n"].concat() +} + +/// Create an isolated host advertising verified audio support for ingress tests. +async fn audio_node() -> mesh::Node { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Worker) + .await + .unwrap(); + node.set_hosted_models(vec![MODEL.into()]).await; + node.set_served_model_descriptors(vec![mesh::ServedModelDescriptor { + identity: mesh::ServedModelIdentity { + model_name: MODEL.into(), + ..Default::default() + }, + capabilities_known: true, + capabilities: crate::models::ModelCapabilities { + audio: crate::models::CapabilityLevel::Supported, + multimodal: true, + ..Default::default() + }, + metadata: Some(mesh::ServedModelMetadata { + workload_class: Some(mesh::ModelWorkloadClass::CausalGeneration), + ..Default::default() + }), + ..Default::default() + }]) + .await; + node +} + +/// Exercise ingress-to-backend forwarding with either fixed or chunked framing. +async fn route_audio(path: &str, chunked: bool) { + let backend = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let backend_port = backend.local_addr().unwrap().port(); + let backend_task = tokio::spawn(async move { + let (mut stream, _) = backend.accept().await.unwrap(); + let request = proxy::read_http_request(&mut stream).await.unwrap(); + stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 13\r\nConnection: close\r\n\r\n{\"text\":\"ok\"}").await.unwrap(); + request + }); + let node = audio_node().await; + let mut targets = election::ModelTargets::default(); + targets.targets.insert( + MODEL.into(), + vec![election::InferenceTarget::Local(backend_port)], + ); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let ingress = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + handle_api_proxy_connection( + node, + stream.into(), + targets, + affinity::AffinityRouter::new(), + crate::runtime::IngressType::LocalOpenAi, + ) + .await; + }); + let body = multipart("auto"); + let framing = if chunked { + "Transfer-Encoding: chunked".into() + } else { + format!("Content-Length: {}", body.len()) + }; + let mut raw = format!("POST {path} HTTP/1.1\r\nHost: localhost\r\nContent-Type: multipart/form-data; boundary=test\r\n{framing}\r\n\r\n").into_bytes(); + if chunked { + for chunk in body.chunks(11) { + raw.extend_from_slice(format!("{:x}\r\n", chunk.len()).as_bytes()); + raw.extend_from_slice(chunk); + raw.extend_from_slice(b"\r\n"); + } + raw.extend_from_slice(b"0\r\n\r\n"); + } else { + raw.extend_from_slice(&body); + } + let mut client = TcpStream::connect(address).await.unwrap(); + client.write_all(&raw).await.unwrap(); + let mut response = String::new(); + client.read_to_string(&mut response).await.unwrap(); + assert!(response.starts_with("HTTP/1.1 200"), "{response}"); + let forwarded = backend_task.await.unwrap(); + ingress.await.unwrap(); + assert_eq!(forwarded.model_name.as_deref(), Some(MODEL)); + assert_eq!( + forwarded.body_bytes.as_deref(), + Some(multipart(MODEL).as_slice()) + ); + let header_end = forwarded + .raw + .windows(4) + .position(|bytes| bytes == b"\r\n\r\n") + .unwrap(); + let headers = String::from_utf8_lossy(&forwarded.raw[..header_end]).to_lowercase(); + assert!(!headers.contains("transfer-encoding:")); + assert!(headers.contains(&format!("content-length: {}", multipart(MODEL).len()))); +} + +#[tokio::test] +/// Pin binary preservation across automatic routing and HTTP body reconstruction. +async fn automatic_audio_reaches_http_backend_with_intact_binary_and_decoded_framing() { + for path in ["/v1/audio/transcriptions", "/v1/audio/translations?trace=1"] { + for chunked in [false, true] { + tokio::time::timeout( + std::time::Duration::from_secs(10), + route_audio(path, chunked), + ) + .await + .expect("audio route must finish"); + } + } +} diff --git a/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/automatic_routing.rs b/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/automatic_routing.rs index 76be59f11e..448bd6c67a 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/automatic_routing.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/automatic_routing.rs @@ -15,6 +15,36 @@ use super::super::ingress::{ AutoRouteResolution, prepare_cache_routing_body, resolve_auto_routed_model, }; +#[tokio::test] +/// A same-name local chat model must not hide a compatible remote embedding replica. +async fn auto_readiness_uses_remote_embedding_despite_local_causal_copy() { + let model = "shared-workload-model"; + let (node, targets) = node_serving(&[model]).await; + node.set_served_model_descriptors(vec![workload_descriptor( + model, + mesh::ModelWorkloadClass::CausalGeneration, + )]) + .await; + let peer_id = iroh::SecretKey::generate().public(); + let mut peer = peer_serving(peer_id, model, false); + peer.served_model_descriptors = vec![workload_descriptor( + model, + mesh::ModelWorkloadClass::Embedding, + )]; + node.insert_test_peer(peer).await; + assert!( + super::super::ingress::auto_route_model_has_ready_ingress_target( + &node, + &targets, + model, + None, + "/v1/embeddings", + &affinity::AffinityRouter::new() + ) + .await + ); +} + /// A served model with the given capabilities, ready for the media filter. fn descriptor(model: &str, vision: bool, audio: bool) -> mesh::ServedModelDescriptor { use crate::models::CapabilityLevel; @@ -43,6 +73,34 @@ fn descriptor(model: &str, vision: bool, audio: bool) -> mesh::ServedModelDescri } } +/// Construct an explicit workload advertisement without inferring support from a name. +fn workload_descriptor( + model: &str, + workload_class: mesh::ModelWorkloadClass, +) -> mesh::ServedModelDescriptor { + mesh::ServedModelDescriptor { + metadata: Some(mesh::ServedModelMetadata { + workload_class: Some(workload_class), + ..Default::default() + }), + ..descriptor(model, false, false) + } +} + +/// Keep audio capability attached to the descriptor that owns the audio workload. +fn audio_workload_descriptor( + model: &str, + workload_class: mesh::ModelWorkloadClass, +) -> mesh::ServedModelDescriptor { + mesh::ServedModelDescriptor { + metadata: Some(mesh::ServedModelMetadata { + workload_class: Some(workload_class), + ..Default::default() + }), + ..descriptor(model, false, true) + } +} + fn request_with_body(model: Option<&str>, body: &serde_json::Value) -> proxy::BufferedHttpRequest { let body = serde_json::to_vec(body).expect("serialize body"); let raw = format!( @@ -145,6 +203,277 @@ async fn resolve( .await } +/// Drive automatic ingress selection for a concrete endpoint and optional model. +async fn resolve_path( + path: &str, + model: Option<&str>, + body: &serde_json::Value, + node: &mesh::Node, + targets: &election::ModelTargets, + descriptors: &[mesh::ServedModelDescriptor], +) -> AutoRouteResolution { + let mut request = request_with_body(model, body); + request.path = path.to_string(); + request.client_path = path.to_string(); + let affinity = affinity::AffinityRouter::new(); + resolve_auto_routed_model( + node, + &mut request, + targets, + None, + descriptors, + None, + &affinity, + ) + .await +} + +#[test] +/// Pin endpoint admission so new routes cannot silently inherit chat-only routing. +fn endpoint_paths_map_to_their_required_workload_classes() { + assert_eq!( + super::super::ingress::request_workload_class("/v1/embeddings?trace=1"), + Some(mesh::ModelWorkloadClass::Embedding) + ); + assert_eq!( + super::super::ingress::request_workload_class("/v1/rerank"), + Some(mesh::ModelWorkloadClass::Rerank) + ); + assert_eq!( + super::super::ingress::request_workload_class("/v1/audio/speech"), + Some(mesh::ModelWorkloadClass::SpeechSynthesis) + ); + assert_eq!( + super::super::ingress::request_workload_class("/v1/audio/transcriptions"), + Some(mesh::ModelWorkloadClass::CausalGeneration) + ); + assert_eq!( + super::super::ingress::request_workload_class("/v1/audio/translations?trace=1"), + Some(mesh::ModelWorkloadClass::CausalGeneration) + ); +} + +#[tokio::test] +/// Require a matching audio workload, not merely a model with a familiar identifier. +async fn audio_upload_routes_only_to_an_advertised_audio_workload() { + let (node, targets) = + node_serving(&["legacy-audio", "chat-only", "tts", "audio-to-text"]).await; + let descriptors = vec![ + descriptor("legacy-audio", false, true), + workload_descriptor("chat-only", mesh::ModelWorkloadClass::CausalGeneration), + audio_workload_descriptor("tts", mesh::ModelWorkloadClass::SpeechSynthesis), + audio_workload_descriptor("audio-to-text", mesh::ModelWorkloadClass::CausalGeneration), + ]; + for path in ["/v1/audio/transcriptions", "/v1/audio/translations?trace=1"] { + for requested_model in [automatic::DIRECTIVE, "audio-to-text"] { + let body = serde_json::json!({ "model": requested_model }); + let resolution = resolve_path( + path, + Some(requested_model), + &body, + &node, + &targets, + &descriptors, + ) + .await; + match resolution { + AutoRouteResolution::Continue { + effective_model, .. + } => assert_eq!(effective_model.as_deref(), Some("audio-to-text")), + AutoRouteResolution::WorkloadUnsupported(workload) => { + panic!("advertised audio-to-text model was rejected for {workload:?}") + } + AutoRouteResolution::MediaUnsupported => { + panic!("advertised audio-to-text model was rejected for media") + } + } + } + } +} + +#[tokio::test] +/// Legacy audio metadata cannot certify the newly introduced upload endpoints. +async fn audio_upload_rejects_legacy_audio_for_explicit_and_auto_routing() { + let (node, targets) = node_serving(&["legacy-audio"]).await; + let descriptors = vec![descriptor("legacy-audio", false, true)]; + + for path in ["/v1/audio/transcriptions", "/v1/audio/translations"] { + for requested_model in ["legacy-audio", automatic::DIRECTIVE] { + let body = serde_json::json!({ "model": requested_model }); + let resolution = resolve_path( + path, + Some(requested_model), + &body, + &node, + &targets, + &descriptors, + ) + .await; + assert!(matches!( + resolution, + AutoRouteResolution::WorkloadUnsupported( + mesh::ModelWorkloadClass::CausalGeneration + ) + )); + } + } +} + +#[tokio::test] +/// A workload label alone must not manufacture native audio decoding support. +async fn audio_upload_requires_runtime_verified_audio_capability() { + let (node, targets) = node_serving(&["unverified-audio", "text-only"]).await; + let mut unverified = audio_workload_descriptor( + "unverified-audio", + mesh::ModelWorkloadClass::CausalGeneration, + ); + unverified.capabilities_known = false; + let descriptors = vec![ + unverified, + workload_descriptor("text-only", mesh::ModelWorkloadClass::CausalGeneration), + ]; + let body = serde_json::json!({ "model": automatic::DIRECTIVE }); + + let resolution = resolve_path( + "/v1/audio/transcriptions", + Some(automatic::DIRECTIVE), + &body, + &node, + &targets, + &descriptors, + ) + .await; + assert!(matches!( + resolution, + AutoRouteResolution::WorkloadUnsupported(mesh::ModelWorkloadClass::CausalGeneration) + )); +} + +#[tokio::test] +/// Preserve existing chat-media routing while tightening the new upload routes. +async fn legacy_chat_audio_remains_routable() { + let (node, targets) = node_serving(&["legacy-audio"]).await; + let descriptors = vec![descriptor("legacy-audio", false, true)]; + let body = serde_json::json!({ + "model": automatic::DIRECTIVE, + "messages": [{ + "role": "user", + "content": [{ + "type": "input_audio", + "input_audio": { "data": "AA==", "format": "wav" }, + }], + }], + }); + + let resolution = resolve_path( + "/v1/chat/completions", + Some(automatic::DIRECTIVE), + &body, + &node, + &targets, + &descriptors, + ) + .await; + match resolution { + AutoRouteResolution::Continue { + effective_model, .. + } => assert_eq!(effective_model.as_deref(), Some("legacy-audio")), + AutoRouteResolution::WorkloadUnsupported(workload) => { + panic!("legacy chat-audio was rejected for {workload:?}") + } + AutoRouteResolution::MediaUnsupported => panic!("legacy model advertises audio input"), + } +} + +#[tokio::test] +/// Automatic embedding selection excludes otherwise healthy chat candidates. +async fn embedding_auto_route_selects_only_an_embedding_model() { + let (node, targets) = node_serving(&["chat-model", "embed-model"]).await; + let descriptors = vec![ + workload_descriptor("chat-model", mesh::ModelWorkloadClass::CausalGeneration), + workload_descriptor("embed-model", mesh::ModelWorkloadClass::Embedding), + ]; + let body = serde_json::json!({ + "model": automatic::DIRECTIVE, + "input": ["alpha", "beta"], + }); + + let resolution = resolve_path( + "/v1/embeddings", + Some(automatic::DIRECTIVE), + &body, + &node, + &targets, + &descriptors, + ) + .await; + + match resolution { + AutoRouteResolution::Continue { + effective_model, .. + } => assert_eq!(effective_model.as_deref(), Some("embed-model")), + AutoRouteResolution::WorkloadUnsupported(workload) => { + panic!("embedding model was advertised but {workload:?} was rejected") + } + AutoRouteResolution::MediaUnsupported => panic!("embedding request has no media"), + } +} + +#[tokio::test] +/// An absent workload field preserves legacy chat, not unverified non-chat support. +async fn non_chat_auto_route_fails_closed_for_legacy_descriptors() { + let (node, targets) = node_serving(&["legacy-chat-model"]).await; + let descriptors = vec![descriptor("legacy-chat-model", false, false)]; + let body = serde_json::json!({ + "model": automatic::DIRECTIVE, + "input": "alpha", + }); + + let resolution = resolve_path( + "/v1/embeddings", + Some(automatic::DIRECTIVE), + &body, + &node, + &targets, + &descriptors, + ) + .await; + + assert!(matches!( + resolution, + AutoRouteResolution::WorkloadUnsupported(mesh::ModelWorkloadClass::Embedding) + )); +} + +#[tokio::test] +/// Explicit model selection does not bypass endpoint-specific workload admission. +async fn explicitly_named_model_must_advertise_the_endpoint_workload() { + let (node, targets) = node_serving(&["chat-model"]).await; + let descriptors = vec![workload_descriptor( + "chat-model", + mesh::ModelWorkloadClass::CausalGeneration, + )]; + let body = serde_json::json!({ + "model": "chat-model", + "input": "alpha", + }); + + let resolution = resolve_path( + "/v1/embeddings", + Some("chat-model"), + &body, + &node, + &targets, + &descriptors, + ) + .await; + + assert!(matches!( + resolution, + AutoRouteResolution::WorkloadUnsupported(mesh::ModelWorkloadClass::Embedding) + )); +} + #[tokio::test] async fn plain_text_directive_stays_on_the_committee() { let (node, targets) = node_serving(&["vision-model", "text-model"]).await; @@ -167,6 +496,9 @@ async fn plain_text_directive_stays_on_the_committee() { AutoRouteResolution::Continue { effective_model, .. } => assert_eq!(effective_model.as_deref(), Some(automatic::DIRECTIVE)), + AutoRouteResolution::WorkloadUnsupported(workload) => { + panic!("generation request unexpectedly rejected for {workload:?}") + } AutoRouteResolution::MediaUnsupported => panic!("text request is not a media failure"), } } @@ -199,6 +531,9 @@ async fn image_request_resolves_to_a_vision_capable_model() { Some("vision-model"), "an image request must resolve to the vision-capable model, not the directive" ), + AutoRouteResolution::WorkloadUnsupported(workload) => { + panic!("generation request unexpectedly rejected for {workload:?}") + } AutoRouteResolution::MediaUnsupported => { panic!("a vision-capable model is served, so this must not fail") } @@ -252,6 +587,9 @@ async fn deprecated_alias_behaves_exactly_like_the_directive() { AutoRouteResolution::Continue { effective_model, .. } => assert_eq!(effective_model.as_deref(), Some(automatic::DIRECTIVE)), + AutoRouteResolution::WorkloadUnsupported(workload) => { + panic!("generation request unexpectedly rejected for {workload:?}") + } AutoRouteResolution::MediaUnsupported => panic!("text request is not a media failure"), } } @@ -292,6 +630,9 @@ async fn streaming_directive_resolves_to_a_single_model() { "must resolve to a served model, got {model}" ); } + AutoRouteResolution::WorkloadUnsupported(workload) => { + panic!("generation request unexpectedly rejected for {workload:?}") + } AutoRouteResolution::MediaUnsupported => panic!("no media in this request"), } } @@ -318,6 +659,9 @@ async fn model_less_request_resolves_to_a_single_model() { "a model-less request must not silently convene a committee" ); } + AutoRouteResolution::WorkloadUnsupported(workload) => { + panic!("generation request unexpectedly rejected for {workload:?}") + } AutoRouteResolution::MediaUnsupported => panic!("no media in this request"), } } @@ -443,6 +787,9 @@ async fn image_request_selects_a_vision_model_served_only_by_a_peer() { Some("remote-vision-model"), "the only vision-capable model is on the peer and must still be chosen" ), + AutoRouteResolution::WorkloadUnsupported(workload) => { + panic!("generation request unexpectedly rejected for {workload:?}") + } AutoRouteResolution::MediaUnsupported => { panic!("a peer serves a vision model, so this must not be refused") } @@ -472,6 +819,9 @@ async fn text_request_still_convenes_a_committee_across_two_nodes() { AutoRouteResolution::Continue { effective_model, .. } => assert_eq!(effective_model.as_deref(), Some(automatic::DIRECTIVE)), + AutoRouteResolution::WorkloadUnsupported(workload) => { + panic!("generation request unexpectedly rejected for {workload:?}") + } AutoRouteResolution::MediaUnsupported => panic!("text request is not a media failure"), } } @@ -526,6 +876,9 @@ async fn an_explicitly_named_model_is_never_reinterpreted() { AutoRouteResolution::Continue { effective_model, .. } => assert_eq!(effective_model.as_deref(), Some("text-model")), + AutoRouteResolution::WorkloadUnsupported(workload) => { + panic!("generation request unexpectedly rejected for {workload:?}") + } AutoRouteResolution::MediaUnsupported => panic!("explicit routing is untouched"), } } @@ -676,6 +1029,9 @@ async fn a_non_chat_endpoint_resolves_to_a_single_model() { "a non-chat request must not convene a committee it cannot fan out" ); } + AutoRouteResolution::WorkloadUnsupported(workload) => { + panic!("generation request unexpectedly rejected for {workload:?}") + } AutoRouteResolution::MediaUnsupported => panic!("no media in this request"), } } diff --git a/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/request_object_cleanup.rs b/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/request_object_cleanup.rs new file mode 100644 index 0000000000..909140532d --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/request_object_cleanup.rs @@ -0,0 +1,140 @@ +//! Request-object ownership ends exactly once when automatic admission fails. + +use super::*; +use crate::plugin::{BridgeFuture, PluginManager, PluginRpcBridge, RpcResult, proto}; +use std::sync::{Arc, Mutex}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; + +#[derive(Default)] +struct CompletionRecorder(Mutex>); + +impl PluginRpcBridge for CompletionRecorder { + /// Record blob completion requests so rejection tests can verify object cleanup. + fn handle_request( + &self, + plugin_name: String, + method: String, + params_json: String, + ) -> BridgeFuture> { + assert_eq!(plugin_name, "blobstore"); + assert_eq!(method, "tools/call"); + let params: serde_json::Value = serde_json::from_str(¶ms_json).unwrap(); + assert_eq!( + params["name"], + crate::plugins::blobstore::COMPLETE_REQUEST_TOOL + ); + let request_id = params["arguments"]["request_id"] + .as_str() + .unwrap() + .to_owned(); + self.0.lock().unwrap().push(request_id.clone()); + Box::pin(async move { + let response = rmcp::model::CallToolResult::structured(serde_json::json!({ + "request_id": request_id, "removed_tokens": 1, "removed_bytes": 4, + })); + Ok(RpcResult { + result_json: serde_json::to_string(&response).unwrap(), + }) + }) + } + + /// Ignore notifications in the request-object cleanup test double. + fn handle_notification(&self, _: String, _: String, _: String) -> BridgeFuture<()> { + Box::pin(async {}) + } +} + +/// Track request-owned blobs through ingress rejection and assert cleanup ownership. +async fn rejected_request_releases_objects(model: &str, media: bool) { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Worker) + .await + .unwrap(); + let recorder = Arc::new(CompletionRecorder::default()); + let manager = PluginManager::for_test_bridge(&["blobstore"], recorder.clone()); + manager.set_test_capability_providers(vec![crate::plugin::PluginCapabilityProvider { + capability: crate::plugins::blobstore::OBJECT_STORE_CAPABILITY.into(), + plugin_name: "blobstore".into(), + plugin_status: "running".into(), + endpoint_id: None, + available: true, + detail: None, + }]); + node.set_plugin_manager(manager.clone()).await; + let mut targets = election::ModelTargets::default(); + targets.targets.insert( + "text-only".into(), + vec![election::InferenceTarget::Local(1)], + ); + node.set_served_model_descriptors(vec![mesh::ServedModelDescriptor { + identity: mesh::ServedModelIdentity { + model_name: "text-only".into(), + ..Default::default() + }, + capabilities_known: true, + metadata: Some(mesh::ServedModelMetadata { + workload_class: Some(mesh::ModelWorkloadClass::CausalGeneration), + ..Default::default() + }), + ..Default::default() + }]) + .await; + let path = if media { + "/v1/chat/completions" + } else { + "/v1/embeddings" + }; + let body = if media { + serde_json::json!({"model": model, "messages": [{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}} + ]}]}) + } else { + serde_json::json!({"model": model, "input": "hello"}) + } + .to_string(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let handler = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut request = proxy::read_http_request(&mut stream).await.unwrap(); + // Parsing/normalization already owns these objects when this handler starts. + request.request_object_request_ids = vec!["upload-a".into(), "upload-b".into()]; + let affinity = affinity::AffinityRouter::new(); + handle_buffered_api_request( + stream.into(), + request, + ProxyConnectionContext { + route: IngressRouteContext { + node: &node, + targets: &targets, + affinity: &affinity, + plugin_manager: Some(&manager), + exchange_channel: None, + }, + }, + None, + crate::runtime::IngressType::LocalOpenAi, + ) + .await; + }); + let mut client = TcpStream::connect(address).await.unwrap(); + client.write_all(format!("POST {path} HTTP/1.1\r\nHost: localhost\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", body.len()).as_bytes()).await.unwrap(); + let mut response = String::new(); + client.read_to_string(&mut response).await.unwrap(); + handler.await.unwrap(); + assert!(response.starts_with("HTTP/1.1 422"), "{response}"); + assert_eq!(*recorder.0.lock().unwrap(), ["upload-a", "upload-b"]); +} + +#[tokio::test] +/// Each rejection path must release every uploaded object exactly once. +async fn workload_and_media_rejections_complete_each_request_object_once() { + for (model, media) in [("text-only", false), ("auto", false), ("auto", true)] { + tokio::time::timeout( + std::time::Duration::from_secs(10), + rejected_request_releases_objects(model, media), + ) + .await + .expect("rejected ingress must complete"); + } +} diff --git a/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/tests.rs b/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/tests.rs index d5275b2da7..036d27693b 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/tests.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/ingress_tests/tests.rs @@ -121,6 +121,7 @@ async fn phantom_model_is_not_auto_route_eligible() { &targets, "phantom/model:Q4_K_M", None, + "/v1/chat/completions", &affinity, ) .await; @@ -149,6 +150,7 @@ async fn freshly_served_local_model_is_auto_route_eligible() { &targets, "local/fresh-model:Q4_K_M", None, + "/v1/chat/completions", &affinity, ) .await; diff --git a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/context_selection.rs b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/context_selection.rs index 14b221832a..af1f1c76b3 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/context_selection.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/context_selection.rs @@ -50,12 +50,14 @@ pub(in crate::network::openai) fn select_degrade_model( .or(unknown) } +/// Keep workload-eligible hosts, preferring known fitting contexts over unknown sizes. pub(super) async fn eligible_remote_hosts( node: &mesh::Node, model: &str, required_tokens: Option, hosts: Vec, ) -> Vec { + let hosts = super::workload_admission::eligible_remote_hosts(node, model, &hosts).await; let Some(required_tokens) = required_tokens else { return hosts; }; diff --git a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/mod.rs b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/mod.rs index ca9daf337b..d5762b7d79 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/mod.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/mod.rs @@ -60,12 +60,10 @@ async fn degrade_to_single_model( let mut candidates = targets .map(super::ingress::callable_models) .unwrap_or_default(); - if candidates.is_empty() { - candidates = node.models_being_served().await; - } - if candidates.is_empty() { - candidates = node.serving_models().await; - } + candidates.extend(node.models_being_served().await); + candidates.extend(node.serving_models().await); + let descriptors = node.all_served_model_descriptors().await; + candidates.retain(|model| workload_admission::model_supports_committee(model, &descriptors)); let runtimes = node.all_model_runtime_descriptors().await; let Some(target) = context_selection::select_degrade_model(candidates, &runtimes, required_tokens) @@ -258,6 +256,7 @@ mod progress; mod self_fill; mod streaming; mod workers; +mod workload_admission; async fn admitted_gateway_config( node: &mesh::Node, diff --git a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/pool.rs b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/pool.rs index b5843afeb2..a187a84233 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/pool.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/pool.rs @@ -8,6 +8,7 @@ use super::context_selection; use super::self_fill::self_fill_from_extra_instances; use super::workers::{LocalModelBackend, RemoteModelBackend}; +use super::workload_admission; use crate::inference::election; use crate::mesh; use mesh_mixture_of_agents as moa; @@ -36,6 +37,9 @@ enum SizeTier { async fn gossiped_sizes(node: &mesh::Node) -> HashMap { let mut by_base: HashMap = HashMap::new(); for descriptor in node.all_served_model_descriptors().await { + if !workload_admission::descriptor_supports_committee(&descriptor) { + continue; + } if let Some(b) = descriptor .metadata .as_ref() @@ -83,7 +87,11 @@ async fn model_routing_hints( use crate::proto::node::InferenceAdmissionState; let mut hints = HashMap::new(); + let local_descriptors = node.served_model_descriptors().await; for local in node.hosted_models().await { + if !workload_admission::model_supports_committee(&local, &local_descriptors) { + continue; + } hints.insert( canonical_base_name(&local), (AvailabilityRank::Healthy, None), @@ -100,6 +108,10 @@ async fn model_routing_hints( _ => AvailabilityRank::Healthy, }; for model in peer.http_routable_models() { + if !workload_admission::model_supports_committee(&model, &peer.served_model_descriptors) + { + continue; + } let model_base = canonical_base_name(&model); let throughput = peer .advertised_model_throughput @@ -264,7 +276,15 @@ async fn add_worker_backend( }) }) }); - if let Some(port) = local_port { + if let Some(port) = local_port + && !workload_admission::eligible_targets( + resolution.node, + name, + &[election::InferenceTarget::Local(port)], + ) + .await + .is_empty() + { let context_length = resolution.node.local_model_context_length(name).await; if context_selection::context_can_satisfy(resolution.required_tokens, context_length) { let backend_idx = backends.len(); @@ -334,6 +354,7 @@ pub(super) async fn assemble_worker_pool( let mut backends: Vec> = Vec::new(); let mut models: Vec = Vec::new(); let mut local_count = 0usize; + let descriptors = node.all_served_model_descriptors().await; // Full mesh-wide model list (local + every peer's advertised routable // models). @@ -342,6 +363,7 @@ pub(super) async fn assemble_worker_pool( .await .into_iter() .filter(|n| n != moa::VIRTUAL_MODEL_NAME) + .filter(|name| workload_admission::model_supports_committee(name, &descriptors)) .collect(); // Verified sizes gossiped by peers (metadata.parameter_count_b). The @@ -641,6 +663,9 @@ pub(super) async fn compute_actor_candidates( let mut tool_use_by_base: std::collections::HashMap = std::collections::HashMap::new(); for descriptor in node.all_served_model_descriptors().await { + if !workload_admission::descriptor_supports_committee(&descriptor) { + continue; + } let base = canonical_base_name(&descriptor.identity.model_name); let level = descriptor.capabilities.tool_use; tool_use_by_base diff --git a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/self_fill.rs b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/self_fill.rs index f94c01332c..cbfb893c76 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/self_fill.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/self_fill.rs @@ -20,6 +20,8 @@ async fn select_clones( ) -> Vec<(InferenceTarget, Option)> { use crate::proto::node::InferenceAdmissionState; + let candidates = super::workload_admission::eligible_targets(node, name, &candidates).await; + let deprioritized: std::collections::HashSet<_> = node .peers() .await diff --git a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/workload_admission.rs b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/workload_admission.rs new file mode 100644 index 0000000000..a3c1469249 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/workload_admission.rs @@ -0,0 +1,88 @@ +//! Every MoA role sends chat completions, including tool actors and reducers. +//! Encoder-decoder support for standalone generation does not certify that +//! conversational/tool contract. Exclude it, as well as stateless workloads, +//! from every committee role until a separate role capability is defined. +//! Legacy peers without workload metadata retain their historical chat role. + +use crate::inference::election::InferenceTarget; +use crate::mesh::{self, ModelWorkloadClass, ServedModelDescriptor}; + +#[cfg(test)] +mod tests; + +/// Accept causal chat or legacy metadata, not standalone encoder-decoder support. +pub(super) fn descriptor_supports_committee(descriptor: &ServedModelDescriptor) -> bool { + matches!( + descriptor + .metadata + .as_ref() + .and_then(|metadata| metadata.workload_class), + None | Some(ModelWorkloadClass::CausalGeneration) + ) +} + +/// Resolve model aliases before deciding whether any descriptor admits chat roles. +pub(super) fn model_supports_committee(model: &str, descriptors: &[ServedModelDescriptor]) -> bool { + let mut matching = descriptors + .iter() + .filter(|descriptor| { + crate::mesh::model_identity::descriptor_matches_routable_name(descriptor, model) + }) + .peekable(); + matching.peek().is_none() || matching.any(descriptor_supports_committee) +} + +/// Admission is target-local: another peer's descriptor cannot authorize this +/// endpoint. Apply before context ranking, reservations, and standby selection. +pub(super) async fn eligible_targets( + node: &mesh::Node, + model: &str, + candidates: &[InferenceTarget], +) -> Vec { + let local = node.served_model_descriptors().await; + let state = node.state.lock().await; + candidates + .iter() + .filter(|target| { + let descriptors = match target { + InferenceTarget::Local(_) => local.as_slice(), + InferenceTarget::Remote(id) => { + let Some(peer) = state.peers.get(id) else { + // A committee reserves work on known members. Unlike + // direct legacy routing, a vanished peer only shrinks + // the pool; it must not receive a committee role. + return false; + }; + peer.served_model_descriptors.as_slice() + } + InferenceTarget::None => return false, + }; + model_supports_committee(model, descriptors) + }) + .cloned() + .collect() +} + +/// Apply committee admission to each known peer without borrowing another's class. +pub(super) async fn eligible_remote_hosts( + node: &mesh::Node, + model: &str, + hosts: &[iroh::EndpointId], +) -> Vec { + let targets = hosts + .iter() + .copied() + .map(InferenceTarget::Remote) + .collect::>(); + eligible_targets(node, model, &targets) + .await + .into_iter() + .filter_map(|target| { + if let InferenceTarget::Remote(id) = target { + Some(id) + } else { + None + } + }) + .collect() +} diff --git a/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/workload_admission/tests.rs b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/workload_admission/tests.rs new file mode 100644 index 0000000000..a8a789d420 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/network/openai/moa_gateway/workload_admission/tests.rs @@ -0,0 +1,158 @@ +use super::super::fleet_sim_tests::{BIG_MODELS, SMALL_MODELS, fleet_peer}; +use super::super::pool::{assemble_worker_pool, compute_actor_candidates}; +use super::*; +use crate::inference::election::ModelTargets; +use crate::network::affinity::AffinityRouter; + +#[test] +/// Future unknown classes cannot inherit the compatibility allowance for old chat peers. +fn explicit_unknown_workload_is_not_a_legacy_committee_member() { + let peer = classified_peer(1, ModelWorkloadClass::Unknown); + assert!(!descriptor_supports_committee( + &peer.served_model_descriptors[0] + )); + assert!(!model_supports_committee( + BIG_MODELS[0].name, + &peer.served_model_descriptors + )); +} + +/// Create one peer whose served descriptor explicitly declares its workload class. +fn classified_peer(seed: u32, class: ModelWorkloadClass) -> mesh::PeerInfo { + let mut peer = fleet_peer(seed, BIG_MODELS[0]); + peer.served_model_descriptors[0] + .metadata + .as_mut() + .unwrap() + .workload_class = Some(class); + peer +} + +#[tokio::test] +/// Filter both committee roles before non-chat models can participate in generation. +async fn mixed_workload_fleet_only_admits_chat_models_to_worker_and_actor_roles() { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Client) + .await + .unwrap(); + let classes = [ + ModelWorkloadClass::CausalGeneration, + ModelWorkloadClass::Embedding, + ModelWorkloadClass::Rerank, + ModelWorkloadClass::EncoderDecoder, + ModelWorkloadClass::SpeechSynthesis, + ]; + for (index, class) in classes.into_iter().enumerate() { + let model = if index < 3 { + BIG_MODELS[index] + } else { + SMALL_MODELS[index - 3] + }; + let mut peer = fleet_peer(index as u32 + 1, model); + peer.served_model_descriptors[0] + .metadata + .as_mut() + .unwrap() + .workload_class = Some(class); + node.insert_test_peer(peer).await; + } + let (backends, models) = + assemble_worker_pool(&node, None, None, &reqwest::Client::new(), None).await; + assert_eq!(backends.len(), 1); + assert_eq!(models.len(), 1); + assert_eq!( + super::super::pool::canonical_base_name(&models[0].name), + super::super::pool::canonical_base_name(BIG_MODELS[0].name) + ); + assert_eq!(compute_actor_candidates(&node, &models).await, vec![0]); +} + +#[tokio::test] +/// Workload admission precedes capacity reservations for standbys and replicas. +async fn target_admission_filters_standbys_and_same_model_clones_before_reservations() { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Worker) + .await + .unwrap(); + let model = BIG_MODELS[0].name; + let local = classified_peer(1, ModelWorkloadClass::EncoderDecoder); + node.set_served_model_descriptors(local.served_model_descriptors) + .await; + let mut targets = ModelTargets::default(); + targets + .targets + .insert(model.into(), vec![InferenceTarget::Local(9337)]); + let mut candidates = vec![InferenceTarget::Local(9337)]; + for (seed, class) in [ + (2, ModelWorkloadClass::Embedding), + (3, ModelWorkloadClass::CausalGeneration), + ] { + let peer = classified_peer(seed, class); + candidates.push(InferenceTarget::Remote(peer.id)); + node.insert_test_peer(peer).await; + } + let eligible = eligible_targets(&node, model, &candidates).await; + assert_eq!(eligible, vec![candidates[2].clone()]); + let remote = match candidates[1] { + InferenceTarget::Remote(id) => id, + _ => unreachable!(), + }; + assert!( + super::super::context_selection::eligible_remote_hosts(&node, model, None, vec![remote]) + .await + .is_empty() + ); + let affinity = AffinityRouter::new(); + let (backends, models) = assemble_worker_pool( + &node, + Some(&targets), + None, + &reqwest::Client::new(), + Some(&affinity), + ) + .await; + assert_eq!( + models.len(), + 1, + "incompatible local/remote copies must not fabricate a committee" + ); + assert_eq!(backends.len(), 1); +} + +#[test] +/// Direct generation compatibility does not grant encoder-decoder committee membership. +fn legacy_chat_is_preserved_but_encoder_decoder_never_inherits_a_committee_role() { + assert!(model_supports_committee("legacy", &[])); + let mut descriptor = ServedModelDescriptor::default(); + descriptor.identity.model_name = "model".into(); + assert!(model_supports_committee( + "model", + std::slice::from_ref(&descriptor) + )); + descriptor.metadata = Some(mesh::ServedModelMetadata { + workload_class: Some(ModelWorkloadClass::EncoderDecoder), + ..Default::default() + }); + assert!(!model_supports_committee("model", &[descriptor])); +} + +#[test] +/// Aliases retain the workload restrictions of the descriptor they identify. +fn public_alias_does_not_bypass_non_chat_admission() { + let mut peer = classified_peer(9, ModelWorkloadClass::Embedding); + let descriptor = &mut peer.served_model_descriptors[0]; + descriptor.identity.source_kind = mesh::ModelSourceKind::HuggingFace; + descriptor.identity.repository = Some("fixture/embedding-GGUF".into()); + descriptor.identity.artifact = Some("embedding.Q8_0.gguf".into()); + let alias = peer.public_model_id_for_routable_model(BIG_MODELS[0].name); + assert_ne!(alias, BIG_MODELS[0].name); + assert!(!model_supports_committee( + &alias, + &peer.served_model_descriptors + )); + assert!( + crate::network::openai::workload_routing::model_satisfies_workload_class( + &alias, + ModelWorkloadClass::Embedding, + &peer.served_model_descriptors + ) + ); +} diff --git a/crates/mesh-llm-host-runtime/src/network/openai/mod.rs b/crates/mesh-llm-host-runtime/src/network/openai/mod.rs index 7bfe9d2a32..8525657683 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/mod.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/mod.rs @@ -16,3 +16,4 @@ mod routing_rank; pub(crate) mod runtime_events; mod tool_call_ids; pub(crate) mod transport; +mod workload_routing; diff --git a/crates/mesh-llm-host-runtime/src/network/openai/request_parse.rs b/crates/mesh-llm-host-runtime/src/network/openai/request_parse.rs index 996957a219..63e857af49 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/request_parse.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/request_parse.rs @@ -10,6 +10,11 @@ use super::request_normalize::{ }; use super::routing_rank::descriptor_for_model; +mod audio_multipart; +use audio_multipart::multipart_model_field; +mod body_rewrite; +pub use body_rewrite::{inject_mesh_hooks_flag, rewrite_model_field}; + pub(crate) const MAX_HEADER_BYTES: usize = 64 * 1024; /// Private lifecycle ownership assertion used only on trusted mesh forwarding. /// @@ -20,8 +25,10 @@ pub(crate) const MAX_HEADER_BYTES: usize = 64 * 1024; pub(crate) const RAW_LIFECYCLE_OWNER_HEADER: &str = "x-mesh-llm-raw-lifecycle"; pub(super) const MAX_BODY_BYTES: usize = 8 * 1024 * 1024; const MAX_OBJECT_UPLOAD_BODY_BYTES: usize = 64 * 1024 * 1024; +const MAX_AUDIO_UPLOAD_BODY_BYTES: usize = 64 * 1024 * 1024 + 64 * 1024; const MAX_CHUNKED_WIRE_BYTES: usize = MAX_BODY_BYTES * 6 + 64 * 1024; const MAX_OBJECT_UPLOAD_CHUNKED_WIRE_BYTES: usize = MAX_OBJECT_UPLOAD_BODY_BYTES * 6 + 64 * 1024; +const MAX_AUDIO_UPLOAD_CHUNKED_WIRE_BYTES: usize = MAX_AUDIO_UPLOAD_BODY_BYTES * 6 + 64 * 1024; pub(super) const MAX_HEADERS: usize = 64; const CRLF: &[u8] = b"\r\n"; const LF: &[u8] = b"\n"; @@ -48,6 +55,7 @@ struct ParsedHeaders { path: String, request_id: RequestId, content_length: Option, + content_type: Option, is_chunked: bool, expects_continue: bool, correlation_id: Option, @@ -138,6 +146,12 @@ impl BufferedHttpRequest { is_tokenize_request(&self.method, &self.path) } + /// Multipart audio bytes are encoded media, not prompt text. The proxy + /// cannot infer their eventual model context size from the wire length. + pub fn is_audio_upload_request(&self) -> bool { + self.method == "POST" && is_audio_upload_path(&self.client_path) + } + pub fn ensure_body_json(&mut self) { if self.body_json.is_none() && !self.body_json_attempted { self.body_json = self @@ -354,6 +368,12 @@ where .map_err(|error| OpenAiRequestReadError::after_headers(error, &parsed))? .to_owned(), ) + } else if is_audio_upload_path(&parsed.path) { + match parsed.content_type.as_deref() { + Some(content_type) => multipart_model_field(content_type, &body) + .map_err(|error| OpenAiRequestReadError::after_headers(error, &parsed))?, + None => None, + } } else { metadata.as_ref().and_then(|value| value.model.clone()) }; @@ -536,11 +556,25 @@ fn body_limits_for_path(path: &str, default: HttpReadLimits) -> HttpReadLimits { max_body_bytes: MAX_OBJECT_UPLOAD_BODY_BYTES, max_chunked_wire_bytes: MAX_OBJECT_UPLOAD_CHUNKED_WIRE_BYTES, } + } else if is_audio_upload_path(path_only) { + HttpReadLimits { + max_header_bytes: default.max_header_bytes, + max_body_bytes: MAX_AUDIO_UPLOAD_BODY_BYTES, + max_chunked_wire_bytes: MAX_AUDIO_UPLOAD_CHUNKED_WIRE_BYTES, + } } else { default } } +/// Identify multipart audio endpoints before attempting JSON parsing. +fn is_audio_upload_path(path: &str) -> bool { + matches!( + path.split('?').next().unwrap_or(path), + "/v1/audio/transcriptions" | "/v1/audio/translations" + ) +} + fn finalize_forwarded_request( mut raw: Vec, header_end: usize, @@ -651,6 +685,7 @@ where let mut is_chunked = false; let mut expects_continue = false; let mut correlation_id = None; + let mut content_type = None; for header in req.headers.iter() { if header.name.eq_ignore_ascii_case("content-length") { @@ -671,6 +706,8 @@ where expects_continue = val .split(',') .any(|part| part.trim().eq_ignore_ascii_case("100-continue")); + } else if header.name.eq_ignore_ascii_case("content-type") { + content_type = std::str::from_utf8(header.value).ok().map(str::to_string); } else if header.name.eq_ignore_ascii_case("x-correlation-id") || header.name.eq_ignore_ascii_case("x-request-id") || header.name.eq_ignore_ascii_case("correlation-id") @@ -692,6 +729,7 @@ where path, request_id: request_id_from_headers(req.headers), content_length, + content_type, is_chunked, expects_continue, correlation_id, @@ -955,98 +993,6 @@ pub(super) fn parse_json_body_from_http_request(raw: &[u8]) -> Option, enabled: bool) { - let Some(header_end) = raw.windows(4).position(|w| w == b"\r\n\r\n").map(|i| i + 4) else { - return; - }; - let body = &raw[header_end..]; - let Some(brace) = body.iter().position(|&b| b == b'{') else { - return; - }; - - // Build new body with mesh_hooks injected after opening brace - let fragment = if enabled { - &b"\"mesh_hooks\":true,"[..] - } else { - &b"\"mesh_hooks\":false,"[..] - }; - let mut new_body = Vec::with_capacity(body.len() + fragment.len()); - new_body.extend_from_slice(&body[..brace + 1]); - new_body.extend_from_slice(fragment); - new_body.extend_from_slice(&body[brace + 1..]); - - // Rebuild headers with correct Content-Length - let headers = std::str::from_utf8(&raw[..header_end - 4]).unwrap_or(""); - let mut rebuilt = String::new(); - for line in headers.split("\r\n") { - if line.to_ascii_lowercase().starts_with("content-length:") { - rebuilt.push_str(&format!("Content-Length: {}", new_body.len())); - } else { - rebuilt.push_str(line); - } - rebuilt.push_str("\r\n"); - } - rebuilt.push_str("\r\n"); - - let mut result = rebuilt.into_bytes(); - result.extend_from_slice(&new_body); - *raw = result; -} - -/// Rewrite the JSON body `model` field and rebuild Content-Length. -pub fn rewrite_model_field(request: &mut BufferedHttpRequest, model: &str) { - let Some(header_end) = request - .raw - .windows(4) - .position(|w| w == b"\r\n\r\n") - .map(|i| i + 4) - else { - return; - }; - - let Ok(mut body) = serde_json::from_slice::(&request.raw[header_end..]) - else { - return; - }; - let Some(object) = body.as_object_mut() else { - return; - }; - - object.insert( - "model".to_string(), - serde_json::Value::String(model.to_string()), - ); - let Ok(new_body) = serde_json::to_vec(&body) else { - return; - }; - - let headers = std::str::from_utf8(&request.raw[..header_end - 4]).unwrap_or(""); - let mut rebuilt = String::new(); - for line in headers.split("\r\n") { - if line.to_ascii_lowercase().starts_with("content-length:") { - rebuilt.push_str(&format!("Content-Length: {}", new_body.len())); - } else { - rebuilt.push_str(line); - } - rebuilt.push_str("\r\n"); - } - rebuilt.push_str("\r\n"); - - let mut raw = rebuilt.into_bytes(); - raw.extend_from_slice(&new_body); - - request.raw = raw; - request.body_len_bytes = new_body.len(); - request.body_bytes = Some(new_body); - request.body_json = Some(body); - request.body_json_attempted = true; - request.model_name = Some(model.to_string()); -} - pub fn is_models_list_request(method: &str, path: &str) -> bool { let path = path.split('?').next().unwrap_or(path); method == "GET" && (path == "/v1/models" || path == "/models") diff --git a/crates/mesh-llm-host-runtime/src/network/openai/request_parse/audio_multipart.rs b/crates/mesh-llm-host-runtime/src/network/openai/request_parse/audio_multipart.rs new file mode 100644 index 0000000000..305a8ca6d8 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/network/openai/request_parse/audio_multipart.rs @@ -0,0 +1,190 @@ +use anyhow::{Context, Result, bail}; + +use super::{CRLF, CRLF_HEADER_TERMINATOR, MAX_HEADER_BYTES}; + +/// Extract an HTTP form boundary only if it satisfies the bounded ASCII grammar. +pub(super) fn multipart_boundary(content_type: &str) -> Option<&str> { + let mut parts = content_type.split(';'); + if !parts + .next()? + .trim() + .eq_ignore_ascii_case("multipart/form-data") + { + return None; + } + let boundary = parts.find_map(|part| { + let (name, value) = part.trim().split_once('=')?; + name.trim() + .eq_ignore_ascii_case("boundary") + .then_some(value.trim().trim_matches('"')) + })?; + let valid = !boundary.is_empty() + && boundary.len() <= 70 + && boundary + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"'()+_,-./:=?".contains(&byte)); + valid.then_some(boundary) +} + +/// Find a nonempty byte marker without decoding the surrounding media payload. +fn find_subslice(haystack: &[u8], needle: &[u8]) -> Option { + (!needle.is_empty()) + .then(|| { + haystack + .windows(needle.len()) + .position(|window| window == needle) + }) + .flatten() +} + +/// Ignore marker-like file bytes unless followed by a valid delimiter suffix. +fn find_multipart_boundary(body: &[u8], marker: &[u8], from: usize) -> Option { + let mut cursor = from; + while let Some(offset) = find_subslice(&body[cursor..], marker) { + let start = cursor + offset; + let suffix = body.get(start + marker.len()..start + marker.len() + 2); + if suffix == Some(CRLF) || suffix == Some(b"--") { + return Some(start); + } + cursor = start + 1; + } + None +} + +/// Split parameters outside quoted strings and reject unterminated escaping. +fn disposition_parameters(value: &str) -> Result> { + let mut parameters = Vec::new(); + let mut start = 0; + let mut quoted = false; + let mut escaped = false; + for (index, byte) in value.bytes().enumerate() { + if escaped { + escaped = false; + continue; + } + match byte { + b'\\' if quoted => escaped = true, + b'"' => quoted = !quoted, + b';' if !quoted => { + parameters.push(&value[start..index]); + start = index + 1; + } + _ => {} + } + } + if quoted || escaped { + bail!("malformed multipart Content-Disposition"); + } + parameters.push(&value[start..]); + Ok(parameters) +} + +/// Identify the model field without mistaking quoted filename text for its name. +pub(super) fn multipart_part_is_model(headers: &str) -> Result { + let mut disposition = None; + for line in headers.split("\r\n") { + let Some((name, value)) = line.split_once(':') else { + continue; + }; + if name.eq_ignore_ascii_case("content-disposition") + && disposition.replace(value.trim()).is_some() + { + bail!("duplicate multipart Content-Disposition header"); + } + } + let Some(disposition) = disposition else { + return Ok(false); + }; + let parameters = disposition_parameters(disposition)?; + if !parameters[0].trim().eq_ignore_ascii_case("form-data") { + return Ok(false); + } + let mut field_name = None; + for parameter in parameters.iter().skip(1) { + let Some((name, value)) = parameter.trim().split_once('=') else { + continue; + }; + if name.trim().eq_ignore_ascii_case("name") { + if field_name.is_some() { + bail!("duplicate multipart field name parameter"); + } + let value = value.trim(); + let parsed = if let Some(inner) = value + .strip_prefix('"') + .and_then(|quoted| quoted.strip_suffix('"')) + { + inner + } else if !value.contains('"') { + value + } else { + bail!("malformed multipart field name parameter"); + }; + field_name = Some(parsed); + } + } + Ok(field_name == Some("model")) +} + +/// Locate the unique model field while validating framing and bounded part headers. +pub(super) fn multipart_model_value_range( + content_type: &str, + body: &[u8], +) -> Result>> { + let Some(boundary) = multipart_boundary(content_type) else { + return Ok(None); + }; + let delimiter = format!("--{boundary}").into_bytes(); + if !body.starts_with(&delimiter) { + bail!("multipart body must start with its declared boundary"); + } + let next_marker = [CRLF, &delimiter].concat(); + let mut cursor = 0; + let mut model_range = None; + loop { + let part_start = cursor + delimiter.len(); + if body.get(part_start..part_start + 2) == Some(b"--") { + return Ok(model_range); + } + if body.get(part_start..part_start + 2) != Some(CRLF) { + bail!("malformed multipart boundary"); + } + let content_start = part_start + CRLF.len(); + let header_search_end = content_start + .saturating_add(MAX_HEADER_BYTES + CRLF_HEADER_TERMINATOR.len()) + .min(body.len()); + let headers_end = find_subslice( + &body[content_start..header_search_end], + CRLF_HEADER_TERMINATOR, + ) + .map(|offset| content_start + offset) + .context("multipart part is missing a header terminator")?; + if headers_end.saturating_sub(content_start) > MAX_HEADER_BYTES { + bail!("multipart part headers exceed the limit"); + } + let headers = std::str::from_utf8(&body[content_start..headers_end]) + .context("multipart part headers are not UTF-8")?; + let is_model = multipart_part_is_model(headers)?; + let value_start = headers_end + CRLF_HEADER_TERMINATOR.len(); + let value_end = find_multipart_boundary(body, &next_marker, value_start) + .context("multipart part is missing its closing boundary")?; + if is_model { + if model_range.is_some() { + bail!("duplicate multipart model field"); + } + model_range = Some(value_start..value_end); + } + cursor = value_end + CRLF.len(); + } +} + +/// Read the routed model as bounded UTF-8, leaving all uploaded file bytes intact. +pub(super) fn multipart_model_field(content_type: &str, body: &[u8]) -> Result> { + let Some(range) = multipart_model_value_range(content_type, body)? else { + return Ok(None); + }; + let value = std::str::from_utf8(&body[range])?.trim(); + if value.len() > 256 { + bail!("multipart model field exceeds the 256-byte limit"); + } + Ok((!value.is_empty()).then(|| value.to_string())) +} diff --git a/crates/mesh-llm-host-runtime/src/network/openai/request_parse/audio_multipart_tests.rs b/crates/mesh-llm-host-runtime/src/network/openai/request_parse/audio_multipart_tests.rs new file mode 100644 index 0000000000..eb84731f4b --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/network/openai/request_parse/audio_multipart_tests.rs @@ -0,0 +1,199 @@ +//! Multipart routing and byte-preserving request regression tests. + +use super::super::audio_multipart::{multipart_boundary, multipart_part_is_model}; +use super::*; + +#[tokio::test] +/// Model routing must never decode or rewrite the uploaded audio bytes. +async fn multipart_model_is_parsed_and_rewritten_without_touching_file_bytes() { + const BOUNDARY: &str = "mesh-audio-boundary"; + // A boundary prefix inside binary content is not a multipart delimiter. + let file_bytes = [ + 0_u8, 255, 13, 10, b'-', b'-', b'm', b'e', b's', b'h', b'-', b'a', b'u', b'd', b'i', b'o', + b'-', b'b', b'o', b'u', b'n', b'd', b'a', b'r', b'y', b'X', 13, 10, 1, 2, 3, 128, + ]; + let mut body = format!( + "--{BOUNDARY}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"voice; name=model\"\r\nContent-Type: audio/wav\r\n\r\n" + ) + .into_bytes(); + body.extend_from_slice(&file_bytes); + body.extend_from_slice( + format!( + "\r\n--{BOUNDARY}\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\nauto\r\n--{BOUNDARY}--\r\n" + ) + .as_bytes(), + ); + let headers = format!( + "POST /v1/audio/transcriptions HTTP/1.1\r\nHost: localhost\r\nContent-Type: multipart/form-data; boundary=\"{BOUNDARY}\"\r\nContent-Length: {}\r\n\r\n", + body.len() + ) + .into_bytes(); + + let mut request = read_request_from_parts(vec![headers, body]).await; + assert_eq!(request.model_name.as_deref(), Some("auto")); + + rewrite_model_field(&mut request, "whisper-local"); + + assert_eq!(request.model_name.as_deref(), Some("whisper-local")); + assert!( + request + .raw + .windows(file_bytes.len()) + .any(|window| window == file_bytes) + ); + let header_end = request + .raw + .windows(4) + .position(|window| window == b"\r\n\r\n") + .unwrap() + + 4; + let content_type = format!("multipart/form-data; boundary={BOUNDARY}"); + assert_eq!( + multipart_model_field(&content_type, &request.raw[header_end..]) + .unwrap() + .as_deref(), + Some("whisper-local") + ); + let declared = std::str::from_utf8(&request.raw[..header_end]) + .unwrap() + .lines() + .find_map(|line| { + line.split_once(':') + .filter(|(name, _)| name.eq_ignore_ascii_case("content-length")) + .and_then(|(_, value)| value.trim().parse::().ok()) + }) + .unwrap(); + assert_eq!(declared, request.raw.len() - header_end); + assert_eq!(declared, request.body_len_bytes); +} + +#[test] +/// Reject ambiguous framing and unbounded model identifiers before route selection. +fn multipart_parser_rejects_invalid_boundaries_and_oversized_model_values() { + assert!(multipart_boundary("multipart/form-data; boundary=bad space").is_none()); + assert!(multipart_boundary("application/json; boundary=mesh").is_none()); + + let boundary = "mesh"; + let body = format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\n{}\r\n--{boundary}--\r\n", + "x".repeat(257) + ); + let error = multipart_model_field( + &format!("multipart/form-data; boundary={boundary}"), + body.as_bytes(), + ) + .unwrap_err(); + assert!(error.to_string().contains("256-byte limit")); +} + +/// The identifier ceiling counts UTF-8 bytes; absent and blank fields stay optional. +#[test] +fn multipart_model_limit_preserves_valid_and_missing_values() { + let content_type = "multipart/form-data; boundary=mesh"; + for value in ["x".repeat(256), "é".repeat(128), " auto ".to_string()] { + let body = format!( + "--mesh\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\n{value}\r\n--mesh--\r\n" + ); + assert_eq!( + multipart_model_field(content_type, body.as_bytes()).unwrap(), + Some(value.trim().to_string()) + ); + } + for body in [ + b"--mesh--\r\n".as_slice(), + b"--mesh\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\n \t\r\n--mesh--\r\n", + b"--mesh\r\nContent-Disposition: form-data; name=\"file\"; filename=\"audio.wav\"\r\n\r\nWAVE\r\n--mesh--\r\n", + ] { + assert_eq!(multipart_model_field(content_type, body).unwrap(), None); + } +} + +/// Both audio endpoints reject overlong explicit destinations instead of routing automatically. +#[tokio::test] +async fn oversized_multipart_model_is_rejected_before_audio_routing() { + for path in ["/v1/audio/transcriptions", "/v1/audio/translations"] { + for value in ["x".repeat(257), "é".repeat(129)] { + let body = format!( + "--mesh\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\n{value}\r\n--mesh--\r\n" + ); + let request = format!( + "POST {path} HTTP/1.1\r\nContent-Type: multipart/form-data; boundary=mesh\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ); + let (mut client, mut server) = tokio::io::duplex(request.len() + 1); + client.write_all(request.as_bytes()).await.unwrap(); + client.shutdown().await.unwrap(); + let error = read_http_request_with_plugin_manager_with_context(&mut server, None) + .await + .unwrap_err(); + assert_eq!(error.context().unwrap().client_path, path); + assert!(error.to_string().contains("256-byte limit")); + } + } +} + +#[tokio::test] +/// Two model fields cannot disagree about the destination of one upload. +async fn duplicate_multipart_model_is_rejected_before_audio_routing() { + let boundary = "mesh-audio-boundary"; + let body = format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\nauto\r\n\ + --{boundary}\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\nother-model\r\n\ + --{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"voice.wav\"\r\n\r\nWAVE\r\n\ + --{boundary}--\r\n" + ); + let content_type = format!("multipart/form-data; boundary={boundary}"); + assert!( + multipart_model_field(&content_type, body.as_bytes()) + .unwrap_err() + .to_string() + .contains("duplicate multipart model field") + ); + + for path in ["/v1/audio/transcriptions", "/v1/audio/translations"] { + let request = format!( + "POST {path} HTTP/1.1\r\nHost: localhost\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ); + let (mut client, mut server) = tokio::io::duplex(request.len() + 1); + client.write_all(request.as_bytes()).await.unwrap(); + client.shutdown().await.unwrap(); + let error = read_http_request_with_plugin_manager_with_context(&mut server, None) + .await + .unwrap_err(); + assert_eq!(error.context().unwrap().client_path, path); + assert!( + error + .to_string() + .contains("duplicate multipart model field") + ); + } +} + +#[test] +/// A boundary-like sequence inside payload data is not a valid multipart start. +fn multipart_model_scanner_rejects_non_initial_boundary() { + let body = b"binary--mesh\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\nauto\r\n--mesh--\r\n"; + let error = multipart_model_field("multipart/form-data; boundary=mesh", body).unwrap_err(); + assert!( + error + .to_string() + .contains("must start with its declared boundary") + ); +} + +#[test] +/// Quoted filenames cannot impersonate the disposition's model-field parameter. +fn multipart_disposition_ignores_name_like_text_inside_quoted_filename() { + assert!( + !multipart_part_is_model( + "Content-Disposition: form-data; name=\"file\"; filename=\"voice; name=model\"" + ) + .unwrap() + ); + assert!(multipart_part_is_model("Content-Disposition: form-data; name=\"model\"").unwrap()); + assert!( + multipart_part_is_model("Content-Disposition: form-data; name=\"file\"; name=\"model\"") + .is_err() + ); +} diff --git a/crates/mesh-llm-host-runtime/src/network/openai/request_parse/body_rewrite.rs b/crates/mesh-llm-host-runtime/src/network/openai/request_parse/body_rewrite.rs new file mode 100644 index 0000000000..a21c8682b5 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/network/openai/request_parse/body_rewrite.rs @@ -0,0 +1,162 @@ +//! Body transformations operate on decoded payloads, never HTTP chunk framing. + +use super::audio_multipart::multipart_model_value_range; +use super::{BufferedHttpRequest, MAX_BODY_BYTES, MAX_HEADERS, try_decode_chunked_body}; + +#[cfg(test)] +mod tests; + +struct BodyHeaders<'a> { + end: usize, + content_type: Option<&'a str>, + chunked: bool, +} + +impl BodyHeaders<'_> { + /// Allow JSON rewriting only for an absent or application/json media type. + fn permits_json(&self) -> bool { + self.content_type.is_none_or(|value| { + value.split(';').next().is_some_and(|media_type| { + media_type.trim().eq_ignore_ascii_case("application/json") + }) + }) + } +} + +/// Extract framing and media type from complete HTTP request headers. +fn body_headers(raw: &[u8]) -> Option> { + let mut headers = [httparse::EMPTY_HEADER; MAX_HEADERS]; + let mut parsed = httparse::Request::new(&mut headers); + let httparse::Status::Complete(end) = parsed.parse(raw).ok()? else { + return None; + }; + let content_type = parsed + .headers + .iter() + .find(|header| header.name.eq_ignore_ascii_case("content-type")) + .map(|header| std::str::from_utf8(header.value)) + .transpose() + .ok()?; + let chunked = parsed + .headers + .iter() + .any(|header| header.name.eq_ignore_ascii_case("transfer-encoding")); + Some(BodyHeaders { + end, + content_type, + chunked, + }) +} + +/// Replace framing with one Content-Length, preserving all other header bytes. +fn replace_body(raw: &mut Vec, header_end: usize, body: &[u8]) { + let mut rebuilt = Vec::with_capacity(header_end + body.len()); + for line in raw[..header_end].split_inclusive(|byte| *byte == b'\n') { + let name = line.split(|byte| *byte == b':').next().unwrap_or_default(); + if line == b"\r\n" + || line == b"\n" + || name.eq_ignore_ascii_case(b"content-length") + || name.eq_ignore_ascii_case(b"transfer-encoding") + || name.eq_ignore_ascii_case(b"trailer") + { + continue; + } + rebuilt.extend_from_slice(line); + } + rebuilt.extend_from_slice(format!("Content-Length: {}\r\n\r\n", body.len()).as_bytes()); + rebuilt.extend_from_slice(body); + *raw = rebuilt; +} + +/// Set the hook flag only on an actual JSON object. Multipart and binary +/// requests remain byte-identical, even when their media contains `{`. +pub fn inject_mesh_hooks_flag(raw: &mut Vec, enabled: bool) { + let Some(headers) = body_headers(raw) else { + return; + }; + if !headers.permits_json() { + return; + } + let decoded; + let body = if headers.chunked { + let Ok(Some((_, bytes))) = try_decode_chunked_body(&raw[headers.end..], MAX_BODY_BYTES) + else { + return; + }; + decoded = bytes; + &decoded[..] + } else { + &raw[headers.end..] + }; + let Ok(mut json) = serde_json::from_slice::(body) else { + return; + }; + let Some(object) = json.as_object_mut() else { + return; + }; + object.insert("mesh_hooks".into(), enabled.into()); + let Ok(body) = serde_json::to_vec(&json) else { + return; + }; + let end = headers.end; + replace_body(raw, end, &body); +} + +/// Replace request framing and parsed body state together after a JSON rewrite. +fn rebuild_request_body( + request: &mut BufferedHttpRequest, + header_end: usize, + body: Vec, + json: Option, + model: &str, +) { + replace_body(&mut request.raw, header_end, &body); + request.body_len_bytes = body.len(); + request.body_bytes = Some(body); + request.body_json = json; + request.body_json_attempted = true; + request.model_name = Some(model.to_string()); +} + +/// Rewrite the JSON or multipart model field using the reader's decoded body +/// for chunked uploads. Only the model part changes; binary media stays intact. +pub fn rewrite_model_field(request: &mut BufferedHttpRequest, model: &str) { + let Some(headers) = body_headers(&request.raw) else { + return; + }; + let original = if headers.chunked { + let Some(body) = request.body_bytes.as_deref() else { + return; + }; + body + } else { + &request.raw[headers.end..] + }; + if headers.permits_json() { + let Ok(mut json) = serde_json::from_slice::(original) else { + return; + }; + let Some(object) = json.as_object_mut() else { + return; + }; + object.insert("model".into(), model.into()); + let Ok(body) = serde_json::to_vec(&json) else { + return; + }; + let end = headers.end; + rebuild_request_body(request, end, body, Some(json), model); + return; + } + let Some(content_type) = headers.content_type else { + return; + }; + let Ok(Some(range)) = multipart_model_value_range(content_type, original) else { + return; + }; + let mut body = Vec::with_capacity(original.len() - range.len() + model.len()); + body.extend_from_slice(&original[..range.start]); + body.extend_from_slice(model.as_bytes()); + body.extend_from_slice(&original[range.end..]); + let end = headers.end; + rebuild_request_body(request, end, body, None, model); +} diff --git a/crates/mesh-llm-host-runtime/src/network/openai/request_parse/body_rewrite/tests.rs b/crates/mesh-llm-host-runtime/src/network/openai/request_parse/body_rewrite/tests.rs new file mode 100644 index 0000000000..e2121a8da4 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/network/openai/request_parse/body_rewrite/tests.rs @@ -0,0 +1,122 @@ +use super::*; +use tokio::io::AsyncWriteExt; + +/// Parse a synthetic connection through the production buffered HTTP reader. +async fn read_request(raw: Vec) -> BufferedHttpRequest { + let (mut writer, mut reader) = tokio::io::duplex(4096); + let write = tokio::spawn(async move { + writer.write_all(&raw).await.unwrap(); + }); + let request = crate::network::openai::request_parse::read_http_request(&mut reader) + .await + .expect("valid request"); + write.await.unwrap(); + request +} + +/// Construct a model field beside opaque upload bytes for rewrite regression tests. +fn multipart(model: &str) -> Vec { + [ + b"--audio\r\nContent-Disposition: form-data; name=\"file\"; filename=\"audio.wav\"\r\nContent-Type: audio/wav\r\n\r\nRIFF\0{\xff}\x80\r\n--audio\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\n".as_slice(), + model.as_bytes(), b"\r\n--audio--\r\n", + ].concat() +} + +/// Encode equivalent requests with fixed-length or chunked transport bodies. +fn raw_request(path: &str, content_type: &str, body: &[u8], chunked: bool) -> Vec { + let framing = if chunked { + "Transfer-Encoding: chunked\r\nTrailer: X-Checksum".into() + } else { + format!("Content-Length: {}", body.len()) + }; + let mut raw = format!("POST {path} HTTP/1.1\r\nHost: localhost\r\nContent-Type: {content_type}\r\n{framing}\r\n\r\n").into_bytes(); + if chunked { + for chunk in body.chunks(7) { + raw.extend_from_slice(format!("{:x};test=1\r\n", chunk.len()).as_bytes()); + raw.extend_from_slice(chunk); + raw.extend_from_slice(b"\r\n"); + } + raw.extend_from_slice(b"0\r\nX-Checksum: fixture\r\n\r\n"); + } else { + raw.extend_from_slice(body); + } + raw +} + +#[tokio::test] +/// Rewrite only the multipart model field and replace decoded transport framing. +async fn automatic_audio_rewrite_preserves_binary_bytes_and_replaces_chunk_framing() { + for path in ["/v1/audio/transcriptions", "/v1/audio/translations?trace=1"] { + for chunked in [false, true] { + let raw = raw_request( + path, + "multipart/form-data; boundary=audio", + &multipart("auto"), + chunked, + ); + let mut request = read_request(raw).await; + assert_eq!(request.model_name.as_deref(), Some("auto")); + // Host ingress injects first; the passive path injects after rewriting. + let original = request.raw.clone(); + inject_mesh_hooks_flag(&mut request.raw, true); + assert_eq!( + request.raw, original, + "hooks must not modify audio or framing" + ); + rewrite_model_field(&mut request, "audio-model"); + inject_mesh_hooks_flag(&mut request.raw, true); + let headers = body_headers(&request.raw).unwrap(); + assert!(!headers.chunked); + let body = &request.raw[headers.end..]; + assert_eq!(body, multipart("audio-model")); + assert_eq!(request.body_bytes.as_deref(), Some(body)); + assert_eq!(request.body_len_bytes, body.len()); + assert!(request.body_json.is_none()); + let header_text = std::str::from_utf8(&request.raw[..headers.end]) + .unwrap() + .to_lowercase(); + assert_eq!(header_text.matches("content-length:").count(), 1); + assert!(header_text.contains(&format!("content-length: {}\r\n", body.len()))); + assert!(!header_text.contains("trailer:")); + } + } +} + +#[test] +/// JSON hooks must leave non-JSON payloads untouched and avoid duplicate flags. +fn hook_injection_rejects_non_json_and_sets_one_valid_flag() { + for (content_type, body) in [ + ("application/octet-stream", b"{\"binary\":true}".as_slice()), + ("application/json", b"prefix{\"invalid\":true}".as_slice()), + ("application/json", b"[{}]".as_slice()), + ] { + let mut raw = raw_request("/v1/chat/completions", content_type, body, false); + let before = raw.clone(); + inject_mesh_hooks_flag(&mut raw, true); + assert_eq!(raw, before); + } + for body in [ + b"{}".as_slice(), + b"{\"mesh_hooks\":false,\"model\":\"auto\"}".as_slice(), + ] { + for chunked in [false, true] { + let mut raw = raw_request( + "/v1/chat/completions", + "application/json; charset=utf-8", + body, + chunked, + ); + inject_mesh_hooks_flag(&mut raw, true); + let headers = body_headers(&raw).unwrap(); + assert!(!headers.chunked); + let json: serde_json::Value = serde_json::from_slice(&raw[headers.end..]).unwrap(); + assert_eq!(json["mesh_hooks"], true); + assert_eq!( + String::from_utf8_lossy(&raw[headers.end..]) + .matches("mesh_hooks") + .count(), + 1 + ); + } + } +} diff --git a/crates/mesh-llm-host-runtime/src/network/openai/request_parse_tests.rs b/crates/mesh-llm-host-runtime/src/network/openai/request_parse_tests.rs index 85d12e7408..f8e1fb10e3 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/request_parse_tests.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/request_parse_tests.rs @@ -1,4 +1,6 @@ use super::*; +#[path = "request_parse/audio_multipart_tests.rs"] +mod audio_multipart_tests; use tokio::io::AsyncWriteExt; use tokio::net::TcpListener; use tokio::net::TcpStream; @@ -652,6 +654,44 @@ async fn test_read_http_request_allows_large_object_upload_body() { assert!(request.request_object_request_ids.is_empty()); } +#[tokio::test] +/// Audio uploads use their own body ceiling instead of the ordinary JSON limit. +async fn test_read_http_request_allows_large_audio_upload_body() { + let file_bytes = vec![b'x'; MAX_BODY_BYTES + 1]; + let mut body = + b"--audio\r\nContent-Disposition: form-data; name=\"file\"; filename=\"large.wav\"\r\n\r\n" + .to_vec(); + body.extend_from_slice(&file_bytes); + body.extend_from_slice(b"\r\n--audio\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\naudio-model\r\n--audio--\r\n"); + let headers = format!( + "POST /v1/audio/transcriptions HTTP/1.1\r\nHost: localhost\r\nContent-Type: multipart/form-data; boundary=audio\r\nContent-Length: {}\r\n\r\n", + body.len() + ) + .into_bytes(); + + let request = read_request_from_parts(vec![headers, body.clone()]).await; + + assert_eq!(request.path, "/v1/audio/transcriptions"); + assert!(request.raw.ends_with(&body)); + assert_eq!(request.body_len_bytes, body.len()); + assert_eq!(request.model_name.as_deref(), Some("audio-model")); +} + +#[test] +/// Only the upload endpoints receive the larger binary-body budget. +fn audio_upload_limits_are_path_scoped() { + let audio = body_limits_for_path("/v1/audio/translations?trace=1", HTTP_READ_LIMITS); + assert_eq!(audio.max_body_bytes, MAX_AUDIO_UPLOAD_BODY_BYTES); + assert_eq!( + audio.max_chunked_wire_bytes, + MAX_AUDIO_UPLOAD_CHUNKED_WIRE_BYTES + ); + + let embedding = body_limits_for_path("/v1/embeddings", HTTP_READ_LIMITS); + assert_eq!(embedding.max_body_bytes, MAX_BODY_BYTES); + assert_eq!(embedding.max_chunked_wire_bytes, MAX_CHUNKED_WIRE_BYTES); +} + #[tokio::test] async fn test_read_http_request_expect_100_continue() { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); diff --git a/crates/mesh-llm-host-runtime/src/network/openai/response/models.rs b/crates/mesh-llm-host-runtime/src/network/openai/response/models.rs index bce5e1e674..4df1bbed76 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/response/models.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/response/models.rs @@ -140,6 +140,9 @@ fn model_metadata_json( ) -> Option { let mut metadata = serde_json::Map::new(); let descriptor_metadata = descriptor.and_then(|descriptor| descriptor.metadata.as_ref()); + if let Some(value) = descriptor_metadata.and_then(|metadata| metadata.workload_class) { + metadata.insert("workload_class".to_string(), serde_json::json!(value)); + } if let Some(value) = descriptor_metadata.and_then(|metadata| metadata.architecture.as_ref()) { metadata.insert("architecture".to_string(), serde_json::json!(value)); } @@ -411,6 +414,7 @@ mod tests { let models = vec!["Qwen3-32B-Q4_K_M".to_string()]; let mut descriptor = local_gguf_descriptor(&models[0]); descriptor.metadata = Some(mesh::ServedModelMetadata { + workload_class: Some(mesh::ModelWorkloadClass::CausalGeneration), architecture: Some("qwen3".to_string()), parameter_size: Some("32B".to_string()), parameter_count_b: Some(32.0), @@ -433,6 +437,7 @@ mod tests { let body = models_list_json(&models, &[descriptor], &runtimes); let metadata = &body["data"][0]["metadata"]; + assert_eq!(metadata["workload_class"], "causal_generation"); assert_eq!(metadata["architecture"], "qwen3"); assert_eq!(metadata["parameter_size"], "32B"); diff --git a/crates/mesh-llm-host-runtime/src/network/openai/routing_rank.rs b/crates/mesh-llm-host-runtime/src/network/openai/routing_rank.rs index 952cf0603d..8506adb6fe 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/routing_rank.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/routing_rank.rs @@ -358,13 +358,6 @@ pub(crate) fn capabilities_for_model( .unwrap_or_else(|| crate::models::installed_model_capabilities(model)) } -pub(crate) fn descriptor_metadata_for_model<'a>( - model: &str, - descriptors: &'a [mesh::ServedModelDescriptor], -) -> Option<&'a mesh::ServedModelMetadata> { - descriptor_for_model(descriptors, model).and_then(|descriptor| descriptor.metadata.as_ref()) -} - #[cfg(test)] mod tests { use super::*; @@ -391,6 +384,7 @@ mod tests { ..local_gguf_descriptor(model_name) } } + #[test] fn test_cached_auto_model_rejects_text_model_for_image_request() { let body = serde_json::json!({ diff --git a/crates/mesh-llm-host-runtime/src/network/openai/transport.rs b/crates/mesh-llm-host-runtime/src/network/openai/transport.rs index 896b6e9e2f..5e3df0c1da 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/transport.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/transport.rs @@ -15,6 +15,7 @@ use crate::network::affinity::{ use crate::network::openai::auto_route; use crate::network::openai::client_stream::ClientStream; use crate::network::openai::response_quality::ResponseQualityFailure; +use crate::network::openai::workload_routing; use crate::network::router; use std::time::{Duration, Instant}; @@ -29,9 +30,7 @@ pub(crate) use super::response::{ send_400_observed, send_503_observed, send_error_observed, send_json_ok_with_headers, send_json_with_status_and_headers_observed, send_models_list_with_descriptors, }; -pub(crate) use super::routing_rank::{ - capabilities_for_model, descriptor_metadata_for_model, request_budget_tokens_from_parts, -}; +pub(crate) use super::routing_rank::{capabilities_for_model, request_budget_tokens_from_parts}; use super::response::{ CacheCostObservation, ResponseRetryPolicy, RouteAttemptLoggingContext, RouteAttemptResult, @@ -172,11 +171,12 @@ pub(crate) async fn reject_legacy_lifecycle_request( ) } -/// Generation context is a property of decode requests, not capability RPCs. -/// A tokenizer request may carry a megabyte of source text while using no -/// target KV context at all. +/// Generation context cannot be estimated from every request body's byte size. +/// Tokenizer requests use no target KV context, and multipart audio bodies +/// contain encoded media rather than text tokens. The audio backend performs +/// the authoritative media/context validation after routing. pub(crate) fn request_context_budget(request: &BufferedHttpRequest) -> Option { - if request.is_tokenize_request() { + if request.is_tokenize_request() || request.is_audio_upload_request() { None } else { request_budget_tokens_from_parts(request.body_len_bytes, request.completion_tokens) @@ -186,6 +186,7 @@ pub(crate) fn request_context_budget(request: &BufferedHttpRequest) -> Option), UnsupportedMedia, + UnsupportedWorkload, } enum MeshTargetResolution { @@ -206,6 +207,7 @@ struct MeshRequestPlan { enum MeshRequestFailure { UnsupportedMedia, + UnsupportedWorkload, ModelUnavailable(String), NoHostsAvailable, } @@ -508,7 +510,21 @@ async fn build_mesh_request_plan( AutoModelResolution::UnsupportedMedia => { return Err(MeshRequestFailure::UnsupportedMedia); } + AutoModelResolution::UnsupportedWorkload => { + return Err(MeshRequestFailure::UnsupportedWorkload); + } }; + if let Some(model) = effective_model.as_deref() + && let Some(workload) = workload_routing::request_workload_class(&request.client_path) + && !workload_routing::model_satisfies_request_workload( + model, + workload, + &request.client_path, + &descriptors, + ) + { + return Err(MeshRequestFailure::UnsupportedWorkload); + } rewrite_effective_model(request, effective_model.as_deref()); if is_auto_request { inject_mesh_hooks_flag(&mut request.raw, true); @@ -525,6 +541,16 @@ async fn build_mesh_request_plan( MeshTargetResolution::NoHostsAvailable => return Err(MeshRequestFailure::NoHostsAvailable), }; + let resolved_hosts = if let Some(model) = effective_model.as_deref() { + workload_routing::eligible_remote_hosts(node, model, &request.client_path, &resolved_hosts) + .await + } else { + resolved_hosts + }; + if resolved_hosts.is_empty() { + return Err(MeshRequestFailure::UnsupportedWorkload); + } + let mut prepared = prepare_mesh_targets( request, effective_model.as_deref(), @@ -571,7 +597,7 @@ fn prepare_mesh_targets( if !request.is_tokenize_request() && effective_model.is_some() && !target_hosts.is_empty() { request.ensure_body_json(); } - let body_json = request.body_json.as_ref(); + let body_json = workload_routing::affinity_body(request); effective_model .map(|name| prepare_remote_targets_for_request(name, target_hosts, body_json, affinity)) .unwrap_or(PreparedTargets { @@ -651,6 +677,15 @@ async fn handle_mesh_request_failure( ) { let mut tcp_stream = Some(tcp_stream); match failure { + MeshRequestFailure::UnsupportedWorkload => { + let _ = send_error_observed( + tcp_stream.take().unwrap(), + 422, + "no serving target advertises support for the requested workload endpoint", + route_observer, + ) + .await; + } MeshRequestFailure::UnsupportedMedia => { let _ = send_error_observed( tcp_stream.take().unwrap(), @@ -1011,10 +1046,14 @@ fn terminal_outcome_for_mesh_route_result( } } +/// Distinguish rejected workload/media admission from unavailable-model routing failures. fn terminal_outcome_for_mesh_request_failure( failure: &MeshRequestFailure, ) -> crate::logging::TerminalOutcome { match failure { + MeshRequestFailure::UnsupportedWorkload => { + crate::logging::TerminalOutcome::Rejected(Some("unsupported_workload".into())) + } MeshRequestFailure::UnsupportedMedia => { crate::logging::TerminalOutcome::Rejected(Some("unsupported_media".into())) } @@ -1119,11 +1158,12 @@ async fn finish_exhausted_mesh_request( let _ = send_503_observed(tcp_stream, &reason, route_observer).await; } +/// Derive generation affinity only for automatic chat routes, never stateless workloads. fn auto_session_key_for_request( request: &mut BufferedHttpRequest, is_auto_request: bool, ) -> Option { - if !is_auto_request { + if !is_auto_request || !workload_routing::supports_generation_affinity(&request.client_path) { return None; } request.ensure_body_json(); @@ -1159,34 +1199,30 @@ async fn resolve_auto_model_request(args: AutoModelRequestArgs<'_>) -> AutoModel return AutoModelResolution::Model(None); } request.ensure_body_json(); - let Some(body_json) = request.body_json.as_ref() else { + if request.body_json.is_none() && !workload_routing::is_audio_upload_path(&request.client_path) + { return AutoModelResolution::Model(None); - }; - let media = router::media_requirements(body_json); - // Build candidates with observed throughput so pick_model_classified - // can weight by locally-measured tok/s where samples exist. - let routing_metrics = node.routing_metrics(); - let with_caps: Vec> = served - .iter() - .map(|name| { - let caps = capabilities_for_model(name, descriptors); - let (tps_hint, throughput_samples) = routing_metrics - .tps_for_model(name) - .map(|(tps, samples)| (Some(tps), samples)) - .unwrap_or((None, 0)); - router::RoutingCandidate { - name: name.as_str(), - caps, - parameter_count_b: descriptor_metadata_for_model(name, descriptors) - .and_then(|metadata| metadata.parameter_count_b), - tps_hint, - throughput_samples, - } - }) - .collect(); + } + let empty_body = serde_json::Value::Null; + let body_json = request.body_json.as_ref().unwrap_or(&empty_body); + let media = workload_routing::request_media(&request.client_path, request.body_json.as_ref()); + let with_caps = + workload_routing::routing_candidates(node, served, &request.client_path, descriptors); + if with_caps.is_empty() + && workload_routing::request_workload_class(&request.client_path).is_some() + { + return AutoModelResolution::UnsupportedWorkload; + } let available = router::filter_media_compatible_candidates(&with_caps, &media); let ready_models = if let Some(available) = available.as_ref() { - auto_route::ready_remote_models(node, required_tokens, available, affinity).await + auto_route::ready_remote_models( + node, + required_tokens, + &request.client_path, + available, + affinity, + ) + .await } else { Vec::new() }; @@ -1200,7 +1236,12 @@ async fn resolve_auto_model_request(args: AutoModelRequestArgs<'_>) -> AutoModel ) .await { - return AutoModelResolution::Model(Some(model)); + if with_caps.iter().any(|candidate| candidate.name == model) { + return AutoModelResolution::Model(Some(model)); + } + if let Some(key) = auto_session_key { + affinity.forget_auto_model(key); + } } let Some(available) = available else { diff --git a/crates/mesh-llm-host-runtime/src/network/openai/transport_route_model.rs b/crates/mesh-llm-host-runtime/src/network/openai/transport_route_model.rs index adbdaae522..c8a96b5d22 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/transport_route_model.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/transport_route_model.rs @@ -95,8 +95,14 @@ async fn route_model_request_inner(args: RouteModelRequestArgs<'_>) -> RouteDisp } = args; let route_started = Instant::now(); let mut tcp_stream = tcp_stream; - let ranked = - rank_targets_by_context(&node, model, required_tokens, &targets.candidates(model)).await; + let candidates = super::super::workload_routing::ingress_candidates( + &node, + model, + &request.client_path, + targets, + ) + .await; + let ranked = rank_targets_by_context(&node, model, required_tokens, &candidates).await; let ordered_candidates = affinity.route_eligible_candidates(model, &ranked.ordered); if ordered_candidates.is_empty() { record_route_model_unavailable(&node, model, 0); @@ -108,7 +114,8 @@ async fn route_model_request_inner(args: RouteModelRequestArgs<'_>) -> RouteDisp } route_observer.route_selected(Some(model)); - let prefix_hash = crate::network::affinity::cache_prefix_hash(request.body_json.as_ref()); + let affinity_body = super::super::workload_routing::affinity_body(request); + let prefix_hash = crate::network::affinity::cache_prefix_hash(affinity_body); let cache_target = cache_target_for_request(&node, affinity, model, prefix_hash, &ordered_candidates).await; let Some(ReservedModelRoute { @@ -119,7 +126,7 @@ async fn route_model_request_inner(args: RouteModelRequestArgs<'_>) -> RouteDisp targets, &ranked, model, - request.body_json.as_ref(), + affinity_body, affinity, cache_target, ) diff --git a/crates/mesh-llm-host-runtime/src/network/openai/transport_tests.rs b/crates/mesh-llm-host-runtime/src/network/openai/transport_tests.rs index eba00caf8a..ee45f57357 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/transport_tests.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/transport_tests.rs @@ -143,3 +143,5 @@ mod durable_artifacts; mod lifecycle; #[path = "transport_tests/routing.rs"] mod routing; +#[path = "transport_tests/workload_routing.rs"] +mod workload_routing; diff --git a/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/routing.rs b/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/routing.rs index 7b6a8082b6..3ad42c4b95 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/routing.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/routing.rs @@ -65,7 +65,8 @@ impl crate::network::metrics::RoutingTelemetrySink for PromptShapeSink { } } -fn test_peer_serving_model(peer_id: iroh::EndpointId, model: &str) -> mesh::PeerInfo { +/// Construct a reachable peer advertising the supplied model for routing tests. +pub(super) fn test_peer_serving_model(peer_id: iroh::EndpointId, model: &str) -> mesh::PeerInfo { mesh::PeerInfo { id: peer_id, addr: iroh::EndpointAddr { @@ -338,6 +339,56 @@ async fn remote_tokenizer_plan_routes_identity_model_without_context_rejection() Ok(()) } +#[tokio::test] +/// Keep audio payload bytes out of text-context routing estimates. +async fn remote_audio_upload_ignores_encoded_bytes_as_context_tokens() -> Result<()> { + let model = "acme/audio-model:Q4_K_M"; + let peer_id = iroh::EndpointId::from(iroh::SecretKey::generate().public()); + let node = test_node_with_remote_models(&[(model, peer_id)]).await; + let mut peer = test_peer_serving_model(peer_id, model); + peer.served_model_runtime = vec![mesh::ModelRuntimeDescriptor { + model_name: model.to_owned(), + identity_hash: None, + context_length: Some(8_192), + ready: true, + }]; + node.insert_test_peer(peer).await; + + for path in [ + "/v1/audio/transcriptions", + "/v1/audio/translations?verbose=1", + ] { + let mut request = large_tokenize_request(model); + request.path = path.to_owned(); + request.client_path = path.to_owned(); + request.body_len_bytes = 1_048_576; + let mistaken_text_budget = + request_budget_tokens_from_parts(request.body_len_bytes, request.completion_tokens); + assert!(mistaken_text_budget.is_some_and(|tokens| tokens > 8_192)); + assert_eq!(request_context_budget(&request), None); + + let ranked = order_remote_hosts_by_context( + &node, + model, + request_context_budget(&request), + std::slice::from_ref(&peer_id), + ) + .await; + assert_eq!( + ranked, + vec![peer_id], + "encoded audio must not reject {path}" + ); + } + + let mut text_request = large_tokenize_request(model); + text_request.path = "/v1/chat/completions".to_owned(); + text_request.client_path = text_request.path.clone(); + text_request.body_len_bytes = 1_048_576; + assert!(request_context_budget(&text_request).is_some()); + Ok(()) +} + #[test] fn tokenizer_effective_model_cannot_override_authoritative_identity() { let model = "acme/code-model:Q4_K_M"; @@ -451,7 +502,9 @@ async fn cached_auto_model_stays_sticky_when_no_ready_remote_model_exists() -> R router::RoutingCandidate::unscored(cached_model, caps), router::RoutingCandidate::unscored(alternate_model, caps), ]; - let ready_models = auto_route::ready_remote_models(&node, None, &available, &affinity).await; + let ready_models = + auto_route::ready_remote_models(&node, None, "/v1/chat/completions", &available, &affinity) + .await; assert!(ready_models.is_empty()); let cached = lookup_cached_auto_model( diff --git a/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/workload_routing.rs b/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/workload_routing.rs new file mode 100644 index 0000000000..9c61d50ad6 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/network/openai/transport_tests/workload_routing.rs @@ -0,0 +1,411 @@ +use super::routing::test_peer_serving_model; +use super::*; +use crate::mesh::ModelWorkloadClass; +use crate::models::{CapabilityLevel, ModelCapabilities}; +use crate::network::openai::workload_routing; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +const MODEL: &str = "shared-workload-model"; + +#[tokio::test] +/// An incompatible local copy must not shadow a valid same-model remote target. +async fn explicit_ingress_falls_back_to_capable_remote_when_local_workload_is_incompatible() { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Worker) + .await + .expect("node"); + let remote = peer(&node, Some(ModelWorkloadClass::Embedding), false).await; + for class in [None, Some(ModelWorkloadClass::CausalGeneration)] { + node.set_served_model_descriptors(vec![descriptor(class, false)]) + .await; + let mut targets = election::ModelTargets::default(); + targets + .targets + .insert(MODEL.into(), vec![election::InferenceTarget::Local(9337)]); + let selected = + workload_routing::ingress_candidates(&node, MODEL, "/v1/embeddings", &targets).await; + assert_eq!(selected, vec![election::InferenceTarget::Remote(remote)]); + } +} + +#[tokio::test] +/// A user tag on stateless work must not create generation-session affinity. +async fn stateless_user_metadata_does_not_disable_replica_reservation_spreading() { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Client) + .await + .expect("node"); + for (path, class) in [ + ("/v1/embeddings", ModelWorkloadClass::Embedding), + ("/v1/rerank", ModelWorkloadClass::Rerank), + ("/v1/audio/speech", ModelWorkloadClass::SpeechSynthesis), + ] { + let first = peer(&node, Some(class), false).await; + let second = peer(&node, Some(class), false).await; + let affinity = AffinityRouter::new(); + let mut request = request(path, MODEL); + let plan = build_mesh_request_plan(&node, &mut request, false, &affinity) + .await + .unwrap_or_else(|_| panic!("compatible replicas")); + assert!(!plan.affinity_applied); + assert_eq!(plan.equivalent_hosts, 2); + assert!(plan.target_hosts.contains(&first) && plan.target_hosts.contains(&second)); + let (first_hosts, _first_reservation) = reserve_mesh_request_target(&plan, &affinity); + let (second_hosts, _second_reservation) = reserve_mesh_request_target(&plan, &affinity); + assert_ne!( + first_hosts[0], second_hosts[0], + "concurrent stateless requests must spread" + ); + } +} + +/// Build a descriptor with independently selectable workload and audio capability. +fn descriptor(class: Option, audio: bool) -> mesh::ServedModelDescriptor { + mesh::ServedModelDescriptor { + identity: mesh::ServedModelIdentity { + model_name: MODEL.into(), + ..Default::default() + }, + metadata: Some(mesh::ServedModelMetadata { + workload_class: class, + ..Default::default() + }), + capabilities_known: true, + capabilities: ModelCapabilities { + audio: if audio { + CapabilityLevel::Supported + } else { + CapabilityLevel::None + }, + multimodal: audio, + ..Default::default() + }, + ..Default::default() + } +} + +/// Create an isolated peer carrying the exact descriptors needed by a routing case. +async fn peer( + node: &mesh::Node, + class: Option, + audio: bool, +) -> iroh::EndpointId { + let id = iroh::SecretKey::generate().public(); + let mut peer = test_peer_serving_model(id, MODEL); + peer.served_model_descriptors = vec![descriptor(class, audio)]; + node.insert_test_peer(peer).await; + id +} + +/// Construct a parsed endpoint request without invoking a native model. +fn request(path: &str, model: &str) -> BufferedHttpRequest { + let body = serde_json::json!({ + "model": model, + "input": "workload routing regression", + "user": "workload-session", + "prompt_cache_key": "workload-cache", + }); + let bytes = serde_json::to_vec(&body).expect("serialize body"); + BufferedHttpRequest { + raw: format!("POST {path} HTTP/1.1\r\nHost: localhost\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{}", bytes.len(), body).into_bytes(), + method: "POST".into(), + path: path.into(), + client_path: path.into(), + request_id: RequestId::default(), + body_json: Some(body), + body_json_attempted: true, + body_len_bytes: bytes.len(), + body_bytes: Some(bytes), + completion_tokens: None, + stream: None, + model_name: Some(model.into()), + request_object_request_ids: vec![], + response_adapter: ResponseAdapter::None, + correlation_id: None, + } +} + +#[tokio::test] +/// Admission must use the selected target's own descriptor rather than fleet-wide metadata. +async fn non_chat_targets_require_their_own_workload_advertisement() { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Client) + .await + .expect("node"); + let legacy = peer(&node, None, true).await; + for (path, class, audio) in [ + ("/v1/embeddings", ModelWorkloadClass::Embedding, false), + ("/v1/rerank", ModelWorkloadClass::Rerank, false), + ( + "/v1/audio/speech", + ModelWorkloadClass::SpeechSynthesis, + false, + ), + ( + "/v1/audio/transcriptions", + ModelWorkloadClass::CausalGeneration, + true, + ), + ( + "/v1/audio/translations?trace=1", + ModelWorkloadClass::CausalGeneration, + true, + ), + ] { + let capable = peer(&node, Some(class), audio).await; + let candidates = vec![ + election::InferenceTarget::Remote(legacy), + election::InferenceTarget::Remote(capable), + ]; + assert_eq!( + workload_routing::eligible_targets(&node, MODEL, path, &candidates).await, + vec![election::InferenceTarget::Remote(capable)], + "{path} must not inherit another peer's capability", + ); + } +} + +#[tokio::test] +/// Never combine incompatible local and remote declarations into invented support. +async fn local_target_cannot_inherit_a_remote_workload_or_vice_versa() { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Worker) + .await + .expect("node"); + let remote = peer(&node, Some(ModelWorkloadClass::Embedding), false).await; + node.set_served_model_descriptors(vec![descriptor(None, false)]) + .await; + let local = election::InferenceTarget::Local(9337); + let remote_target = election::InferenceTarget::Remote(remote); + let candidates = vec![local.clone(), remote_target.clone()]; + assert_eq!( + workload_routing::eligible_targets(&node, MODEL, "/v1/embeddings", &candidates).await, + vec![remote_target], + ); + + node.set_served_model_descriptors(vec![descriptor(Some(ModelWorkloadClass::Embedding), false)]) + .await; + let mut legacy_peer = test_peer_serving_model(remote, MODEL); + legacy_peer.served_model_descriptors = vec![descriptor(None, false)]; + node.insert_test_peer(legacy_peer).await; + assert_eq!( + workload_routing::eligible_targets(&node, MODEL, "/v1/embeddings", &candidates).await, + vec![local], + ); +} + +#[tokio::test] +/// Workload filtering must preserve existing generation and control-plane behavior. +async fn legacy_generation_and_control_routes_remain_eligible() { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Client) + .await + .expect("node"); + let legacy = peer(&node, None, true).await; + let targets = vec![election::InferenceTarget::Remote(legacy)]; + for path in [ + "/v1/chat/completions", + "/v1/completions", + "/v1/responses", + "/tokenize", + ] { + assert_eq!( + workload_routing::eligible_targets(&node, MODEL, path, &targets).await, + targets + ); + } + node.remove_test_peer(legacy).await; + assert!( + workload_routing::eligible_targets(&node, MODEL, "/v1/embeddings", &targets) + .await + .is_empty() + ); +} + +#[tokio::test] +/// Filter unsupported passive-client targets before assigning routing state. +async fn passive_plan_excludes_legacy_hosts_before_affinity_and_reservation() { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Client) + .await + .expect("node"); + let legacy = peer(&node, None, false).await; + let capable = peer(&node, Some(ModelWorkloadClass::Embedding), false).await; + let affinity = AffinityRouter::new(); + let mut request = request("/v1/embeddings", MODEL); + let prefix = + crate::network::affinity::cache_prefix_hash(request.body_json.as_ref()).expect("prefix"); + affinity.remember_cache_lease_if_epoch( + MODEL, + prefix, + &election::InferenceTarget::Remote(legacy), + affinity.cache_lease_epoch(), + ); + let plan = build_mesh_request_plan(&node, &mut request, false, &affinity) + .await + .unwrap_or_else(|_| panic!("capable peer must remain routable")); + assert_eq!(plan.target_hosts, vec![capable]); + assert_eq!(plan.equivalent_hosts, 1); + let (hosts, _reservation) = reserve_mesh_request_target(&plan, &affinity); + assert_eq!(hosts, vec![capable]); +} + +#[tokio::test] +/// An explicit future class is not an absent legacy descriptor. +async fn passive_plan_rejects_unknown_workloads_instead_of_forwarding_them() { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Client) + .await + .expect("node"); + peer(&node, None, true).await; + let affinity = AffinityRouter::new(); + for path in ["/v1/embeddings", "/v1/audio/transcriptions"] { + let mut request = request(path, MODEL); + assert!(matches!( + build_mesh_request_plan(&node, &mut request, false, &affinity).await, + Err(MeshRequestFailure::UnsupportedWorkload), + )); + } +} + +#[tokio::test] +/// Passive audio routing selects a capable model and preserves the binary upload. +async fn passive_auto_audio_uses_the_capable_descriptor_and_rewrites_multipart() { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Client) + .await + .expect("node"); + peer(&node, None, false).await; + let capable = peer(&node, Some(ModelWorkloadClass::CausalGeneration), true).await; + // A local legacy descriptor deliberately precedes all current peer metadata. + node.set_served_model_descriptors(vec![descriptor(None, false)]) + .await; + let path = "/v1/audio/transcriptions"; + let mut request = request(path, "auto"); + let body = b"--fixture\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\nauto\r\n--fixture\r\nContent-Disposition: form-data; name=\"file\"; filename=\"audio.wav\"\r\n\r\nRIFF\r\n--fixture--\r\n".to_vec(); + request.raw = format!("POST {path} HTTP/1.1\r\nHost: localhost\r\nContent-Type: multipart/form-data; boundary=fixture\r\nContent-Length: {}\r\n\r\n", body.len()).into_bytes(); + request.raw.extend_from_slice(&body); + request.body_len_bytes = body.len(); + request.body_bytes = Some(body); + request.body_json = None; + let plan = build_mesh_request_plan(&node, &mut request, false, &AffinityRouter::new()) + .await + .unwrap_or_else(|_| panic!("current audio peer must be selected")); + assert_eq!(plan.effective_model.as_deref(), Some(MODEL)); + assert_eq!(plan.target_hosts, vec![capable]); + assert!(String::from_utf8_lossy(&request.raw).contains(&format!("\r\n\r\n{MODEL}\r\n"))); + assert!(String::from_utf8_lossy(&request.raw).contains("\r\n\r\nRIFF\r\n")); +} + +#[tokio::test] +/// A cached automatic choice for one endpoint cannot authorize another workload. +async fn passive_auto_model_cache_cannot_cross_workload_boundaries() { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Client) + .await + .expect("node"); + let capable = peer(&node, Some(ModelWorkloadClass::Embedding), false).await; + let chat_id = iroh::SecretKey::generate().public(); + node.insert_test_peer(test_peer_serving_model(chat_id, "legacy-chat")) + .await; + let affinity = AffinityRouter::new(); + let mut request = request("/v1/embeddings", "auto"); + let key = crate::network::affinity::auto_model_session_key(request.body_json.as_ref()) + .expect("legacy cache key"); + assert_eq!(auto_session_key_for_request(&mut request, true), None); + affinity.remember_auto_model(key, "legacy-chat"); + // No healthy compatible alternative: the availability fallback must still + // stay inside the requested workload, never restore the cached chat model. + affinity.record_target_outcome( + Some(MODEL), + &election::InferenceTarget::Remote(capable), + TargetHealthOutcome::Unavailable, + ); + let plan = build_mesh_request_plan(&node, &mut request, false, &affinity) + .await + .unwrap_or_else(|_| panic!("embedding fallback remains available")); + assert_eq!(plan.effective_model.as_deref(), Some(MODEL)); + assert_eq!(plan.target_hosts, vec![capable]); + assert_eq!( + affinity.lookup_auto_model(key).as_deref(), + Some("legacy-chat") + ); +} + +#[tokio::test] +/// Workload and audio support must coexist on one target descriptor. +async fn audio_upload_capabilities_must_belong_to_one_descriptor_on_the_target() { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Client) + .await + .expect("node"); + let id = iroh::SecretKey::generate().public(); + let mut mixed = test_peer_serving_model(id, MODEL); + mixed.served_model_descriptors = vec![ + descriptor(None, true), + descriptor(Some(ModelWorkloadClass::CausalGeneration), false), + ]; + node.insert_test_peer(mixed).await; + let candidates = vec![election::InferenceTarget::Remote(id)]; + assert!( + workload_routing::eligible_targets(&node, MODEL, "/v1/audio/transcriptions", &candidates) + .await + .is_empty() + ); +} + +#[tokio::test] +/// Fail closed when neither the local target nor any replica can serve the endpoint. +async fn host_dispatch_rejects_local_legacy_target_without_capable_replicas() { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Worker) + .await + .expect("node"); + node.set_served_model_descriptors(vec![descriptor(None, false)]) + .await; + let backend = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("backend"); + let backend_port = backend.local_addr().expect("address").port(); + let backend_task = tokio::spawn(async move { + if let Ok(Ok((mut stream, _))) = + tokio::time::timeout(Duration::from_secs(5), backend.accept()).await + { + let mut request_bytes = [0; 4096]; + let _ = stream.read(&mut request_bytes).await; + let _ = stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}") + .await; + } + }); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("client listener"); + let (client, server) = tokio::join!( + tokio::net::TcpStream::connect(listener.local_addr().expect("address")), + listener.accept() + ); + let mut client = client.expect("client"); + let (server, _) = server.expect("server"); + let mut targets = election::ModelTargets::default(); + targets.targets.insert( + MODEL.into(), + vec![election::InferenceTarget::Local(backend_port)], + ); + let request = request("/v1/embeddings", MODEL); + let affinity = AffinityRouter::new(); + let outcome = tokio::time::timeout( + Duration::from_secs(2), + route_model_request( + node, + server.into(), + &targets, + MODEL, + &request, + RouteModelRequestContext { + required_tokens: None, + affinity: &affinity, + route_observer: OpenAiRouteObserver::default(), + }, + ), + ) + .await; + backend_task.abort(); + let _ = backend_task.await; + assert!(matches!(outcome, Ok(RouteDispatchOutcome::Responded(503)))); + let mut response = String::new(); + client + .read_to_string(&mut response) + .await + .expect("response"); + assert!(response.starts_with("HTTP/1.1 503")); +} diff --git a/crates/mesh-llm-host-runtime/src/network/openai/workload_routing.rs b/crates/mesh-llm-host-runtime/src/network/openai/workload_routing.rs new file mode 100644 index 0000000000..00f180d7ca --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/network/openai/workload_routing.rs @@ -0,0 +1,274 @@ +//! Workload admission at the model and individual serving-target boundaries. +//! +//! Model-level admission is existential: any matching advertisement can make a +//! model discoverable. It never grants another peer that model's capabilities. +//! Both host and passive-client dispatch filter their concrete targets before +//! context ranking, affinity, reservations, and retry selection. + +use crate::inference::election::InferenceTarget; +use crate::mesh::model_identity::descriptor_matches_routable_name; +use crate::mesh::{self, ModelWorkloadClass, ServedModelDescriptor}; + +#[cfg(test)] +mod tests; + +/// Recognize binary audio-to-text endpoints independently of query parameters. +pub(super) fn is_audio_upload_path(path: &str) -> bool { + matches!( + path.split('?').next().unwrap_or(path), + "/v1/audio/transcriptions" | "/v1/audio/translations" + ) +} + +/// Map inference endpoints to native workloads; control paths impose no class. +pub(super) fn request_workload_class(path: &str) -> Option { + match path.split('?').next().unwrap_or(path) { + "/v1/chat/completions" + | "/v1/completions" + | "/v1/responses" + | "/v1/audio/transcriptions" + | "/v1/audio/translations" => Some(ModelWorkloadClass::CausalGeneration), + "/v1/embeddings" => Some(ModelWorkloadClass::Embedding), + "/v1/rerank" => Some(ModelWorkloadClass::Rerank), + "/v1/audio/speech" => Some(ModelWorkloadClass::SpeechSynthesis), + _ => None, + } +} + +/// Only generation requests reuse KV/session state. Metadata such as an +/// embeddings `user` field must not pin a stateless workload to one replica. +pub(super) fn supports_generation_affinity(path: &str) -> bool { + matches!( + path.split('?').next().unwrap_or(path), + "/v1/chat/completions" | "/v1/completions" | "/v1/responses" + ) +} + +/// Expose identity hints only for endpoints whose generation state is reusable. +pub(super) fn affinity_body( + request: &super::request_parse::BufferedHttpRequest, +) -> Option<&serde_json::Value> { + supports_generation_affinity(&request.client_path) + .then_some(request.body_json.as_ref()) + .flatten() +} + +/// Prefer the elected targets only when at least one supports this request. +/// A stale/local incompatible copy must not shadow capable remote replicas. +pub(super) async fn ingress_candidates( + node: &mesh::Node, + model: &str, + path: &str, + targets: &crate::inference::election::ModelTargets, +) -> Vec { + let local = eligible_targets(node, model, path, &targets.candidates(model)).await; + if local + .iter() + .any(|target| !matches!(target, InferenceTarget::None)) + { + return local; + } + let remote = node + .hosts_for_model(model) + .await + .into_iter() + .map(InferenceTarget::Remote) + .collect::>(); + eligible_targets(node, model, path, &remote).await +} + +/// Preserve absent legacy chat metadata, but never authorize an unknown class. +fn class_is_compatible( + requested: ModelWorkloadClass, + advertised: Option, +) -> bool { + match (requested, advertised) { + (ModelWorkloadClass::Unknown, _) | (_, Some(ModelWorkloadClass::Unknown)) => false, + (ModelWorkloadClass::CausalGeneration, None) => true, + ( + ModelWorkloadClass::CausalGeneration, + Some(ModelWorkloadClass::CausalGeneration | ModelWorkloadClass::EncoderDecoder), + ) => true, + (requested, Some(advertised)) => requested == advertised, + (_, None) => false, + } +} + +/// Check discoverability across matching descriptors, not target eligibility. +pub(super) fn model_satisfies_workload_class( + model: &str, + requested: ModelWorkloadClass, + descriptors: &[ServedModelDescriptor], +) -> bool { + let mut matching = descriptors + .iter() + .filter(|descriptor| descriptor_matches_routable_name(descriptor, model)) + .peekable(); + if matching.peek().is_none() { + return class_is_compatible(requested, None); + } + matching.any(|descriptor| { + class_is_compatible( + requested, + descriptor + .metadata + .as_ref() + .and_then(|metadata| metadata.workload_class), + ) + }) +} + +/// Require verified audio and a known generation class on the same descriptor. +fn descriptor_supports_audio_upload(descriptor: &ServedModelDescriptor) -> bool { + descriptor.capabilities_known + && descriptor.capabilities.supports_audio_runtime() + && matches!( + descriptor + .metadata + .as_ref() + .and_then(|metadata| metadata.workload_class), + Some(ModelWorkloadClass::CausalGeneration | ModelWorkloadClass::EncoderDecoder) + ) +} + +/// Apply endpoint-specific admission, including stricter audio upload metadata. +pub(super) fn model_satisfies_request_workload( + model: &str, + workload: ModelWorkloadClass, + path: &str, + descriptors: &[ServedModelDescriptor], +) -> bool { + if is_audio_upload_path(path) { + descriptors.iter().any(|descriptor| { + descriptor_matches_routable_name(descriptor, model) + && descriptor_supports_audio_upload(descriptor) + }) + } else { + model_satisfies_workload_class(model, workload, descriptors) + } +} + +/// Select one matching descriptor that independently supports the endpoint. +pub(super) fn descriptor_for_request<'a>( + model: &str, + path: &str, + descriptors: &'a [ServedModelDescriptor], +) -> Option<&'a ServedModelDescriptor> { + descriptors.iter().find(|descriptor| { + descriptor_matches_routable_name(descriptor, model) + && request_workload_class(path).is_none_or(|workload| { + model_satisfies_request_workload( + model, + workload, + path, + std::slice::from_ref(*descriptor), + ) + }) + }) +} + +/// Infer binary upload requirements without interpreting file bytes as JSON. +pub(super) fn request_media( + path: &str, + body: Option<&serde_json::Value>, +) -> crate::network::router::MediaRequirements { + if is_audio_upload_path(path) { + crate::network::router::MediaRequirements { + has_media: true, + needs_audio: true, + needs_vision: false, + } + } else { + body.map_or_else(Default::default, crate::network::router::media_requirements) + } +} + +/// Pick metadata from a descriptor that supports this request, not whichever +/// peer happened to gossip the model name first. +pub(super) fn routing_candidates<'a>( + node: &mesh::Node, + models: &'a [String], + path: &str, + descriptors: &[ServedModelDescriptor], +) -> Vec> { + let metrics = node.routing_metrics(); + models + .iter() + .filter(|model| { + request_workload_class(path).is_none_or(|workload| { + model_satisfies_request_workload(model, workload, path, descriptors) + }) + }) + .map(|model| { + let descriptor = descriptor_for_request(model, path, descriptors); + let caps = descriptor.map_or_else( + || super::routing_rank::capabilities_for_model(model, descriptors), + |descriptor| descriptor.capabilities, + ); + let (tps_hint, throughput_samples) = metrics + .tps_for_model(model) + .map(|(tps, samples)| (Some(tps), samples)) + .unwrap_or((None, 0)); + crate::network::router::RoutingCandidate { + name: model, + caps, + parameter_count_b: descriptor + .and_then(|descriptor| descriptor.metadata.as_ref()) + .and_then(|metadata| metadata.parameter_count_b), + tps_hint, + throughput_samples, + } + }) + .collect() +} + +/// Filter each candidate against its own advertisement before routing decisions. +pub(super) async fn eligible_targets( + node: &mesh::Node, + model: &str, + path: &str, + candidates: &[InferenceTarget], +) -> Vec { + let Some(workload) = request_workload_class(path) else { + return candidates.to_vec(); + }; + let local = node.served_model_descriptors().await; + let state = node.state.lock().await; + candidates + .iter() + .filter(|target| { + let descriptors = match target { + InferenceTarget::Local(_) => local.as_slice(), + InferenceTarget::Remote(peer_id) => state + .peers + .get(peer_id) + .map_or(&[][..], |peer| peer.served_model_descriptors.as_slice()), + InferenceTarget::None => return false, + }; + model_satisfies_request_workload(model, workload, path, descriptors) + }) + .cloned() + .collect() +} + +/// Restrict peer IDs using the same per-target workload policy as local ingress. +pub(super) async fn eligible_remote_hosts( + node: &mesh::Node, + model: &str, + path: &str, + hosts: &[iroh::EndpointId], +) -> Vec { + let candidates = hosts + .iter() + .copied() + .map(InferenceTarget::Remote) + .collect::>(); + eligible_targets(node, model, path, &candidates) + .await + .into_iter() + .filter_map(|target| match target { + InferenceTarget::Remote(peer) => Some(peer), + _ => None, + }) + .collect() +} diff --git a/crates/mesh-llm-host-runtime/src/network/openai/workload_routing/tests.rs b/crates/mesh-llm-host-runtime/src/network/openai/workload_routing/tests.rs new file mode 100644 index 0000000000..ad09f5cc2f --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/network/openai/workload_routing/tests.rs @@ -0,0 +1,145 @@ +use super::*; + +#[test] +/// Distinguish unrecognized explicit metadata from the old absent-field contract. +fn unknown_workloads_never_inherit_legacy_admission() { + let metadata: mesh::ServedModelMetadata = serde_json::from_value(serde_json::json!({ + "workload_class": "future_non_chat_contract" + })) + .unwrap(); + assert_eq!(metadata.workload_class, Some(ModelWorkloadClass::Unknown)); + let descriptors = [mesh::ServedModelDescriptor { + metadata: Some(metadata), + ..local_gguf_descriptor("future") + }]; + for path in [ + "/v1/chat/completions", + "/v1/completions", + "/v1/responses", + "/v1/embeddings", + "/v1/rerank", + "/v1/audio/speech", + "/v1/audio/transcriptions", + "/v1/audio/translations", + ] { + assert!( + !model_satisfies_request_workload( + "future", + request_workload_class(path).unwrap(), + path, + &descriptors + ), + "{path}" + ); + } + let relayed = serde_json::to_value(&descriptors[0].metadata).unwrap(); + assert_eq!(relayed["workload_class"], "unknown"); + let absent: mesh::ServedModelMetadata = serde_json::from_value(serde_json::json!({})).unwrap(); + assert_eq!(absent.workload_class, None); +} + +/// Construct a local artifact identity without implying workload capability. +fn local_gguf_descriptor(model_name: &str) -> ServedModelDescriptor { + ServedModelDescriptor { + identity: mesh::ServedModelIdentity { + model_name: model_name.into(), + ..Default::default() + }, + ..Default::default() + } +} + +/// Attach an explicit class to the same model identity for admission comparisons. +fn descriptor_with_workload( + model_name: &str, + workload_class: mesh::ModelWorkloadClass, +) -> mesh::ServedModelDescriptor { + mesh::ServedModelDescriptor { + metadata: Some(mesh::ServedModelMetadata { + workload_class: Some(workload_class), + ..Default::default() + }), + ..local_gguf_descriptor(model_name) + } +} + +#[test] +/// Preserve old generation peers while rejecting unadvertised non-chat endpoints. +fn legacy_descriptors_are_compatible_only_with_generation_routes() { + let descriptors = vec![local_gguf_descriptor("legacy")]; + + assert!(model_satisfies_workload_class( + "legacy", + mesh::ModelWorkloadClass::CausalGeneration, + &descriptors + )); + assert!(!model_satisfies_workload_class( + "legacy", + mesh::ModelWorkloadClass::Embedding, + &descriptors + )); + assert!(!model_satisfies_workload_class( + "legacy", + mesh::ModelWorkloadClass::SpeechSynthesis, + &descriptors + )); +} + +#[test] +/// A known but different workload never satisfies an endpoint's requirement. +fn workload_routes_require_an_exact_advertised_class() { + let descriptors = vec![ + descriptor_with_workload("embed", mesh::ModelWorkloadClass::Embedding), + descriptor_with_workload("rank", mesh::ModelWorkloadClass::Rerank), + ]; + + assert!(model_satisfies_workload_class( + "embed", + mesh::ModelWorkloadClass::Embedding, + &descriptors + )); + assert!(!model_satisfies_workload_class( + "embed", + mesh::ModelWorkloadClass::Rerank, + &descriptors + )); + assert!(model_satisfies_workload_class( + "rank", + mesh::ModelWorkloadClass::Rerank, + &descriptors + )); +} + +#[test] +/// Reordering equivalent metadata must not change route eligibility. +fn workload_metadata_is_independent_of_descriptor_order() { + let legacy = local_gguf_descriptor("shared-model"); + let current = descriptor_with_workload("shared-model", mesh::ModelWorkloadClass::Embedding); + for descriptors in [vec![legacy.clone(), current.clone()], vec![current, legacy]] { + assert!(model_satisfies_workload_class( + "shared-model", + mesh::ModelWorkloadClass::Embedding, + &descriptors, + )); + } +} + +#[test] +/// Encoder-decoder models retain supported direct generation endpoints. +fn encoder_decoder_models_can_serve_generation_routes() { + let descriptors = vec![descriptor_with_workload( + "t5", + mesh::ModelWorkloadClass::EncoderDecoder, + )]; + + assert!(model_satisfies_workload_class( + "t5", + mesh::ModelWorkloadClass::CausalGeneration, + &descriptors + )); + assert!(!model_satisfies_workload_class( + "t5", + mesh::ModelWorkloadClass::Embedding, + &descriptors + )); +} diff --git a/crates/mesh-llm-host-runtime/src/protocol/convert.rs b/crates/mesh-llm-host-runtime/src/protocol/convert.rs index d4296b6924..40942cf15d 100644 --- a/crates/mesh-llm-host-runtime/src/protocol/convert.rs +++ b/crates/mesh-llm-host-runtime/src/protocol/convert.rs @@ -9,6 +9,8 @@ use anyhow::{Context, Result}; use iroh::{EndpointAddr, EndpointId}; use std::collections::{HashMap, HashSet}; +mod served_descriptors; + fn skippy_stage_subprotocols( artifact_transfer_supported: bool, stage_protocol_generation_supported: bool, @@ -306,22 +308,12 @@ fn proto_identity_to_local( } } -fn legacy_descriptor_from_identity( - identity: &crate::proto::node::ServedModelIdentity, -) -> crate::mesh::ServedModelDescriptor { - crate::mesh::ServedModelDescriptor { - identity: proto_identity_to_local(identity), - capabilities_known: false, - capabilities: crate::models::ModelCapabilities::default(), - topology: None, - metadata: None, - } -} - +/// Encode optional serving metadata without inventing a class for legacy models. fn local_model_metadata_to_proto( metadata: &crate::mesh::ServedModelMetadata, ) -> crate::proto::node::ServedModelMetadata { crate::proto::node::ServedModelMetadata { + workload_class: metadata.workload_class.map(local_workload_class_to_proto), architecture: metadata.architecture.clone(), parameter_size: metadata.parameter_size.clone(), parameter_count_b: metadata.parameter_count_b, @@ -337,10 +329,16 @@ fn local_model_metadata_to_proto( } } +/// Decode serving metadata while distinguishing unknown workload values from absent fields. fn proto_model_metadata_to_local( metadata: &crate::proto::node::ServedModelMetadata, ) -> crate::mesh::ServedModelMetadata { crate::mesh::ServedModelMetadata { + // Keep an explicit unknown descriptor: dropping it can re-enable the + // legacy model-name fallback elsewhere in routing and gossip. + workload_class: metadata + .workload_class + .and_then(proto_workload_class_to_local), architecture: metadata.architecture.clone(), parameter_size: metadata.parameter_size.clone(), parameter_count_b: metadata.parameter_count_b, @@ -356,6 +354,39 @@ fn proto_model_metadata_to_local( } } +/// Encode workload classes using additive protobuf discriminants. +fn local_workload_class_to_proto(workload: crate::mesh::ModelWorkloadClass) -> i32 { + use crate::mesh::ModelWorkloadClass as Local; + use crate::proto::node::ModelWorkloadClass as Proto; + + match workload { + Local::CausalGeneration => Proto::CausalGeneration as i32, + Local::Embedding => Proto::Embedding as i32, + Local::Rerank => Proto::Rerank as i32, + Local::EncoderDecoder => Proto::EncoderDecoder as i32, + Local::SpeechSynthesis => Proto::SpeechSynthesis as i32, + // Preserve explicit denial when relaying metadata. Zero is the legacy + // unspecified value and must not erase an unknown workload. + Local::Unknown => -1, + } +} + +/// Distinguish legacy-unspecified workload metadata from unknown future values. +fn proto_workload_class_to_local(value: i32) -> Option { + use crate::mesh::ModelWorkloadClass as Local; + use crate::proto::node::ModelWorkloadClass as Proto; + + match Proto::try_from(value) { + Ok(Proto::CausalGeneration) => Some(Local::CausalGeneration), + Ok(Proto::Embedding) => Some(Local::Embedding), + Ok(Proto::Rerank) => Some(Local::Rerank), + Ok(Proto::EncoderDecoder) => Some(Local::EncoderDecoder), + Ok(Proto::SpeechSynthesis) => Some(Local::SpeechSynthesis), + Ok(Proto::Unspecified) => None, + Err(_) => Some(Local::Unknown), + } +} + fn runtime_descriptor_to_proto( descriptor: &crate::mesh::ModelRuntimeDescriptor, ) -> crate::proto::node::ModelRuntimeDescriptor { @@ -533,19 +564,6 @@ fn proto_gpu_info_to_legacy_fields(gpus: &[crate::proto::node::GpuInfo]) -> Lega } } -/// Returns `true` when a proto descriptor carries a non-empty model name. -/// Descriptors without a valid identity are discarded so a partial list -/// cannot suppress the legacy-identity backfill fallback. -fn proto_descriptor_has_valid_identity( - descriptor: &crate::proto::node::ServedModelDescriptor, -) -> bool { - descriptor - .identity - .as_ref() - .map(|id| !id.model_name.is_empty()) - .unwrap_or(false) -} - pub(crate) fn sanitize_gossip_announcement_for_wire(ann: &PeerAnnouncement) -> PeerAnnouncement { let mut sanitized = ann.clone(); sanitized.available_models.clear(); @@ -1026,74 +1044,7 @@ pub(crate) fn proto_ann_to_local( .iter() .map(proto_runtime_descriptor_to_local) .collect(), - served_model_descriptors: if !pa.served_model_descriptors.is_empty() { - let descriptors: Vec<_> = - pa.served_model_descriptors - .iter() - .filter(|descriptor| proto_descriptor_has_valid_identity(descriptor)) - .map(|descriptor| { - let capabilities = descriptor - .capabilities - .as_ref() - .map(|caps| crate::models::ModelCapabilities { - multimodal: caps.multimodal, - vision: proto_capability_level_to_local(caps.vision), - audio: proto_capability_level_to_local(caps.audio), - reasoning: proto_capability_level_to_local(caps.reasoning), - tool_use: proto_capability_level_to_local(caps.tool_use), - moe: caps.moe, - }) - .unwrap_or_default(); - crate::mesh::ServedModelDescriptor { - identity: descriptor - .identity - .as_ref() - .map(proto_identity_to_local) - .unwrap_or_default(), - capabilities_known: descriptor.capabilities_known.unwrap_or( - capabilities != crate::models::ModelCapabilities::default(), - ), - capabilities, - topology: descriptor.topology.as_ref().map(|topology| { - crate::models::ModelTopology { - moe: topology.moe.as_ref().map(|moe| { - crate::models::ModelMoeInfo { - expert_count: moe.expert_count, - used_expert_count: moe.used_expert_count, - min_experts_per_node: moe.min_experts_per_node, - source: moe.source.clone(), - ranking_source: moe.ranking_source.clone(), - ranking_origin: moe.ranking_origin.clone(), - ranking: moe.ranking.clone(), - ranking_prompt_count: moe.ranking_prompt_count, - ranking_tokens: moe.ranking_tokens, - ranking_layer_scope: moe.ranking_layer_scope.clone(), - } - }), - } - }), - metadata: descriptor - .metadata - .as_ref() - .map(proto_model_metadata_to_local), - } - }) - .collect(); - if descriptors.is_empty() { - // All descriptors were invalid — fall back to legacy identity list. - pa.served_model_identities - .iter() - .map(legacy_descriptor_from_identity) - .collect() - } else { - descriptors - } - } else { - pa.served_model_identities - .iter() - .map(legacy_descriptor_from_identity) - .collect() - }, + served_model_descriptors: Vec::new(), owner_attestation: pa .owner_attestation .as_ref() @@ -1134,7 +1085,7 @@ pub(crate) fn proto_ann_to_local( .as_ref() .and_then(proto_cache_affinity_to_local), }; - crate::mesh::backfill_legacy_descriptors(&mut ann); + served_descriptors::restore_served_descriptors(pa, &mut ann); ann.advertised_model_throughput = sanitize_model_throughput_hints_for_ann(&ann); ann.cache_affinity = sanitize_cache_affinity_for_ann(&ann); Some((addr, ann)) @@ -1373,6 +1324,94 @@ mod tests { use super::*; use crate::mesh::requirements::peer_release_attestation_status; + #[test] + /// Preserve every supported workload class across protobuf announcement conversion. + fn workload_class_round_trips_through_additive_proto_metadata() { + for workload in [ + crate::mesh::ModelWorkloadClass::CausalGeneration, + crate::mesh::ModelWorkloadClass::Embedding, + crate::mesh::ModelWorkloadClass::Rerank, + crate::mesh::ModelWorkloadClass::EncoderDecoder, + crate::mesh::ModelWorkloadClass::SpeechSynthesis, + crate::mesh::ModelWorkloadClass::Unknown, + ] { + let local = crate::mesh::ServedModelMetadata { + workload_class: Some(workload), + architecture: Some("test".to_string()), + ..Default::default() + }; + + let proto = local_model_metadata_to_proto(&local); + let restored = proto_model_metadata_to_local(&proto); + + assert_eq!(restored.workload_class, Some(workload)); + assert_eq!(restored.architecture.as_deref(), Some("test")); + } + } + + #[test] + /// Accept legacy unspecified classes while rejecting unknown modern classes. + fn absent_proto_workload_class_is_legacy_compatible_but_unknown_is_rejected() { + let absent = crate::proto::node::ServedModelMetadata::default(); + assert_eq!(proto_model_metadata_to_local(&absent).workload_class, None); + let unspecified = crate::proto::node::ServedModelMetadata { + workload_class: Some(crate::proto::node::ModelWorkloadClass::Unspecified as i32), + ..Default::default() + }; + assert_eq!( + proto_model_metadata_to_local(&unspecified).workload_class, + None + ); + + let unknown = crate::proto::node::ServedModelMetadata { + workload_class: Some(9_999), + ..Default::default() + }; + let local = proto_model_metadata_to_local(&unknown); + assert_eq!( + local.workload_class, + Some(crate::mesh::ModelWorkloadClass::Unknown) + ); + let relayed = local_model_metadata_to_proto(&local); + assert_eq!( + proto_model_metadata_to_local(&relayed).workload_class, + local.workload_class + ); + } + + #[test] + /// Prevent unsupported workload descriptors from regaining routes through legacy identities. + fn unknown_descriptor_workload_class_does_not_fall_back_to_legacy_identity() { + let identity = crate::proto::node::ServedModelIdentity { + model_name: "future-workload".to_string(), + ..Default::default() + }; + let proto = crate::proto::node::PeerAnnouncement { + endpoint_id: vec![1; 32], + served_model_identities: vec![identity.clone()], + served_model_descriptors: vec![crate::proto::node::ServedModelDescriptor { + identity: Some(identity), + metadata: Some(crate::proto::node::ServedModelMetadata { + workload_class: Some(9_999), + ..Default::default() + }), + ..Default::default() + }], + ..Default::default() + }; + + let (_, announcement) = proto_ann_to_local(&proto).expect("announcement should decode"); + assert_eq!(announcement.served_model_descriptors.len(), 1); + assert_eq!( + announcement.served_model_descriptors[0] + .metadata + .as_ref() + .unwrap() + .workload_class, + Some(crate::mesh::ModelWorkloadClass::Unknown) + ); + } + #[test] fn proto_ann_to_local_preserves_malformed_release_attestation_for_later_rejection() { let proto = crate::proto::node::PeerAnnouncement { diff --git a/crates/mesh-llm-host-runtime/src/protocol/convert/served_descriptors.rs b/crates/mesh-llm-host-runtime/src/protocol/convert/served_descriptors.rs new file mode 100644 index 0000000000..17855dfad1 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/protocol/convert/served_descriptors.rs @@ -0,0 +1,191 @@ +//! Restore authoritative model descriptors without reviving rejected legacy routes. + +use super::{ + proto_capability_level_to_local, proto_identity_to_local, proto_model_metadata_to_local, +}; +use crate::mesh::{PeerAnnouncement, ServedModelDescriptor}; +use crate::proto::node; + +/// Only peers that omit descriptors may use identity/name-based legacy inference. +pub(super) fn restore_served_descriptors( + source: &node::PeerAnnouncement, + ann: &mut PeerAnnouncement, +) { + if source.served_model_descriptors.is_empty() { + ann.served_model_descriptors = source + .served_model_identities + .iter() + .map(legacy_descriptor_from_identity) + .collect(); + crate::mesh::backfill_legacy_descriptors(ann); + } else { + ann.served_model_descriptors = source + .served_model_descriptors + .iter() + .filter_map(proto_descriptor_to_local) + .collect(); + } +} + +/// Convert a legacy identity without inventing capability or workload metadata. +fn legacy_descriptor_from_identity( + identity: &crate::proto::node::ServedModelIdentity, +) -> crate::mesh::ServedModelDescriptor { + crate::mesh::ServedModelDescriptor { + identity: proto_identity_to_local(identity), + capabilities_known: false, + capabilities: crate::models::ModelCapabilities::default(), + topology: None, + metadata: None, + } +} + +/// Retain complete metadata for valid descriptors and discard missing/empty identities. +fn proto_descriptor_to_local( + descriptor: &node::ServedModelDescriptor, +) -> Option { + let identity = descriptor + .identity + .as_ref() + .filter(|id| !id.model_name.is_empty())?; + let capabilities = descriptor + .capabilities + .as_ref() + .map(|caps| crate::models::ModelCapabilities { + multimodal: caps.multimodal, + vision: proto_capability_level_to_local(caps.vision), + audio: proto_capability_level_to_local(caps.audio), + reasoning: proto_capability_level_to_local(caps.reasoning), + tool_use: proto_capability_level_to_local(caps.tool_use), + moe: caps.moe, + }) + .unwrap_or_default(); + Some(ServedModelDescriptor { + identity: proto_identity_to_local(identity), + capabilities_known: descriptor + .capabilities_known + .unwrap_or(capabilities != crate::models::ModelCapabilities::default()), + capabilities, + topology: descriptor + .topology + .as_ref() + .map(|topology| crate::models::ModelTopology { + moe: topology + .moe + .as_ref() + .map(|moe| crate::models::ModelMoeInfo { + expert_count: moe.expert_count, + used_expert_count: moe.used_expert_count, + min_experts_per_node: moe.min_experts_per_node, + source: moe.source.clone(), + ranking_source: moe.ranking_source.clone(), + ranking_origin: moe.ranking_origin.clone(), + ranking: moe.ranking.clone(), + ranking_prompt_count: moe.ranking_prompt_count, + ranking_tokens: moe.ranking_tokens, + ranking_layer_scope: moe.ranking_layer_scope.clone(), + }), + }), + metadata: descriptor + .metadata + .as_ref() + .map(proto_model_metadata_to_local), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::protocol::convert::proto_ann_to_local; + + /// Include both legacy fallback sources so malformed descriptors cannot hide the regression. + fn announcement_with_legacy_routes() -> node::PeerAnnouncement { + node::PeerAnnouncement { + endpoint_id: vec![1; 32], + serving_models: vec!["legacy-model".to_string()], + model_source: Some("legacy-model".to_string()), + served_model_identities: vec![node::ServedModelIdentity { + model_name: "legacy-model".to_string(), + ..Default::default() + }], + ..Default::default() + } + } + + /// A present-but-invalid descriptor list must never recover a legacy model route. + #[test] + fn invalid_descriptors_do_not_backfill_from_legacy_names_or_identities() { + for descriptor in [ + node::ServedModelDescriptor::default(), + node::ServedModelDescriptor { + identity: Some(node::ServedModelIdentity::default()), + ..Default::default() + }, + ] { + let source = node::PeerAnnouncement { + served_model_descriptors: vec![descriptor], + ..announcement_with_legacy_routes() + }; + let (_, ann) = proto_ann_to_local(&source).unwrap(); + assert!(ann.served_model_descriptors.is_empty()); + } + } + + /// Older identity-aware peers still restore the identities they actually advertise. + #[test] + fn absent_descriptors_preserve_legacy_identity_fallback() { + let (_, ann) = proto_ann_to_local(&announcement_with_legacy_routes()).unwrap(); + assert_eq!(ann.served_model_descriptors.len(), 1); + let descriptor = &ann.served_model_descriptors[0]; + assert_eq!(descriptor.identity.model_name, "legacy-model"); + assert!(!descriptor.capabilities_known); + assert!(descriptor.metadata.is_none()); + } + + /// Name-only peers retain the original primary-model inference contract. + #[test] + fn absent_descriptors_and_identities_preserve_legacy_name_fallback() { + let source = node::PeerAnnouncement { + served_model_identities: Vec::new(), + ..announcement_with_legacy_routes() + }; + let (_, ann) = proto_ann_to_local(&source).unwrap(); + assert_eq!(ann.served_model_descriptors.len(), 1); + let descriptor = &ann.served_model_descriptors[0]; + assert_eq!(descriptor.identity.model_name, "legacy-model"); + assert!(descriptor.identity.is_primary); + assert!(descriptor.metadata.is_none()); + } + + /// Partial invalidity must not drop valid workload metadata or restore extra legacy names. + #[test] + fn valid_descriptors_remain_authoritative_among_invalid_entries() { + let source = node::PeerAnnouncement { + served_model_descriptors: vec![ + node::ServedModelDescriptor::default(), + node::ServedModelDescriptor { + identity: Some(node::ServedModelIdentity { + model_name: "embedding-model".to_string(), + ..Default::default() + }), + capabilities_known: Some(true), + metadata: Some(node::ServedModelMetadata { + workload_class: Some(node::ModelWorkloadClass::Embedding as i32), + ..Default::default() + }), + ..Default::default() + }, + ], + ..announcement_with_legacy_routes() + }; + let (_, ann) = proto_ann_to_local(&source).unwrap(); + assert_eq!(ann.served_model_descriptors.len(), 1); + let descriptor = &ann.served_model_descriptors[0]; + assert_eq!(descriptor.identity.model_name, "embedding-model"); + assert!(descriptor.capabilities_known); + assert_eq!( + descriptor.metadata.as_ref().unwrap().workload_class, + Some(crate::mesh::ModelWorkloadClass::Embedding) + ); + } +} diff --git a/crates/mesh-llm-host-runtime/src/runtime/local.rs b/crates/mesh-llm-host-runtime/src/runtime/local.rs index e96df964be..669b028cf1 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/local.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/local.rs @@ -97,6 +97,7 @@ pub(super) struct LocalRuntimeModelHandle { pub(super) context_length: u32, pub(super) slots: usize, pub(super) capabilities: models::ModelCapabilities, + pub(super) workload_class: mesh::ModelWorkloadClass, pub(super) inner: LocalRuntimeBackendHandle, } @@ -563,11 +564,13 @@ fn weights_digest_toctou_recheck_passes( after.is_some() && after == before } +/// Publish runtime-probed capabilities and workload class without losing identity updates. pub(super) async fn set_runtime_verified_served_model_capabilities( node: &mesh::Node, primary_model_name: &str, model_name: &str, capabilities: models::ModelCapabilities, + workload_class: mesh::ModelWorkloadClass, ) { node.update_served_model_descriptor(model_name, |existing| { runtime_verified_served_model_descriptor( @@ -575,16 +578,20 @@ pub(super) async fn set_runtime_verified_served_model_capabilities( primary_model_name, model_name, capabilities, + workload_class, ) }) .await; } +/// Preserve existing model identity while replacing inferred capabilities with runtime facts. +/// Missing descriptors receive a fallback identity before their workload is advertised. pub(super) fn runtime_verified_served_model_descriptor( existing: Option, primary_model_name: &str, model_name: &str, capabilities: models::ModelCapabilities, + workload_class: mesh::ModelWorkloadClass, ) -> mesh::ServedModelDescriptor { let mut descriptor = existing.unwrap_or_else(|| mesh::ServedModelDescriptor { identity: mesh::ServedModelIdentity { @@ -604,6 +611,10 @@ pub(super) fn runtime_verified_served_model_descriptor( descriptor.capabilities_known = true; descriptor.capabilities = capabilities; descriptor + .metadata + .get_or_insert_with(Default::default) + .workload_class = Some(workload_class); + descriptor } pub(super) async fn remove_serving_assignment(node: &mesh::Node, model_name: &str) { @@ -986,6 +997,7 @@ async fn start_local_skippy_model( }) .await .context("join load skippy direct GGUF task")??; + let workload_class = skippy_model.workload_class()?; let _ = emit_event(OutputEvent::ModelLoaded { model: model_name.clone(), bytes: None, @@ -1001,6 +1013,7 @@ async fn start_local_skippy_model( context_length, slots: plan.slots, capabilities, + workload_class, inner: LocalRuntimeBackendHandle::Skippy { model: skippy_model, http, @@ -1152,6 +1165,7 @@ async fn start_local_package_v2_model( }) .await .context("join load skippy package-v2 task")??; + let workload_class = handle.workload_class()?; let _ = emit_event(OutputEvent::ModelLoaded { model: model_ref, bytes: None, @@ -1167,6 +1181,7 @@ async fn start_local_package_v2_model( context_length, slots: plan.slots, capabilities, + workload_class, inner: LocalRuntimeBackendHandle::Skippy { model: handle, http, @@ -1228,6 +1243,10 @@ pub(super) fn local_process_snapshot( } } +#[cfg(test)] +#[path = "local/descriptor_tests.rs"] +mod descriptor_tests; + #[cfg(test)] mod tests { use super::{ diff --git a/crates/mesh-llm-host-runtime/src/runtime/local/descriptor_tests.rs b/crates/mesh-llm-host-runtime/src/runtime/local/descriptor_tests.rs new file mode 100644 index 0000000000..a260d32d78 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/runtime/local/descriptor_tests.rs @@ -0,0 +1,61 @@ +use crate::{mesh, models}; + +/// Runtime facts and the generation-bound file digest can arrive in either order. +#[tokio::test] +async fn workload_publication_and_weights_digest_preserve_each_other() { + for workload_class in [ + mesh::ModelWorkloadClass::CausalGeneration, + mesh::ModelWorkloadClass::Embedding, + mesh::ModelWorkloadClass::SpeechSynthesis, + ] { + for digest_first in [true, false] { + let node = mesh::Node::new_for_tests(mesh::NodeRole::Worker) + .await + .expect("isolated test node"); + let model_name = "runtime-workload"; + let generation = node.begin_served_model_generation(model_name).await; + let capabilities = models::ModelCapabilities::default(); + let publish_workload = || { + super::set_runtime_verified_served_model_capabilities( + &node, + model_name, + model_name, + capabilities, + workload_class, + ) + }; + if !digest_first { + publish_workload().await; + } + assert!( + super::set_local_model_weights_digest( + &node, + model_name, + generation, + Some("sha256:verified-weights".into()), + ) + .await + ); + if digest_first { + publish_workload().await; + } + + let descriptors = node.served_model_descriptors().await; + let descriptor = descriptors + .iter() + .find(|descriptor| descriptor.identity.model_name == model_name) + .expect("runtime descriptor"); + assert_eq!( + descriptor.identity.weights_digest.as_deref(), + Some("sha256:verified-weights") + ); + assert!(descriptor.identity.is_primary); + assert!(descriptor.capabilities_known); + assert_eq!(descriptor.capabilities, capabilities); + assert_eq!( + descriptor.metadata.as_ref().unwrap().workload_class, + Some(workload_class) + ); + } + } +} diff --git a/crates/mesh-llm-host-runtime/src/runtime/local_split/loading.rs b/crates/mesh-llm-host-runtime/src/runtime/local_split/loading.rs index f27f090b32..55916820b8 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/local_split/loading.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/local_split/loading.rs @@ -359,6 +359,7 @@ pub(super) async fn load_split_runtime_generation_inner( context_length: spec.ctx_size, slots: spec.slots, capabilities, + workload_class: mesh::ModelWorkloadClass::CausalGeneration, inner: LocalRuntimeBackendHandle::Skippy { model: handle, http, diff --git a/crates/mesh-llm-host-runtime/src/runtime/local_split/test_support.rs b/crates/mesh-llm-host-runtime/src/runtime/local_split/test_support.rs index 2f39ce8371..3a08010074 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/local_split/test_support.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/local_split/test_support.rs @@ -818,6 +818,7 @@ fn runtime_verified_served_model_descriptor_preserves_identity_and_updates_capab source_kind: mesh::ModelSourceKind::HuggingFace, repository: Some("Qwen/Qwen3-VL-2B-Instruct-GGUF".into()), artifact: Some("Qwen3VL-2B-Instruct-Q4_K_M.gguf".into()), + weights_digest: Some("sha256:verified-weights".into()), ..Default::default() }, capabilities_known: false, @@ -836,6 +837,7 @@ fn runtime_verified_served_model_descriptor_preserves_identity_and_updates_capab "Qwen3VL-2B-Instruct-Q4_K_M", "Qwen3VL-2B-Instruct-Q4_K_M", capabilities, + mesh::ModelWorkloadClass::CausalGeneration, ); assert_eq!( @@ -849,6 +851,14 @@ fn runtime_verified_served_model_descriptor_preserves_identity_and_updates_capab assert!(descriptor.identity.is_primary); assert!(descriptor.capabilities_known); assert_eq!(descriptor.capabilities, capabilities); + assert_eq!( + descriptor.identity.weights_digest.as_deref(), + Some("sha256:verified-weights") + ); + assert_eq!( + descriptor.metadata.unwrap().workload_class, + Some(mesh::ModelWorkloadClass::CausalGeneration) + ); } #[test] @@ -858,6 +868,7 @@ fn runtime_verified_served_model_descriptor_builds_fallback_identity() { "Primary", "Runtime", models::ModelCapabilities::default(), + mesh::ModelWorkloadClass::Embedding, ); assert_eq!(descriptor.identity.model_name, "Runtime"); diff --git a/crates/mesh-llm-host-runtime/src/runtime/model_lifecycle.rs b/crates/mesh-llm-host-runtime/src/runtime/model_lifecycle.rs index 252b25ea90..8d6167726f 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/model_lifecycle.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/model_lifecycle.rs @@ -5,8 +5,8 @@ use super::{ ModelTargetReconciliationCapacityState, ModelTargetReconciliationInput, ModelTargetReconciliationPolicy, ModelTargetReconciliationState, RunAutoRuntimeLoopContext, RunAutoRuntimeState, RuntimeCapacityReservation, RuntimeEvent, RuntimeInstanceRegistry, - RuntimeOperationalEvent, RuntimeOptions, RuntimeUnloadCandidate, RuntimeUnloadOwner, - ShutdownRuntimeLoadedModelsContext, StartupModelSpec, StartupReadyReporter, + RuntimeModelRegistration, RuntimeOperationalEvent, RuntimeOptions, RuntimeUnloadCandidate, + RuntimeUnloadOwner, ShutdownRuntimeLoadedModelsContext, StartupModelSpec, StartupReadyReporter, add_runtime_local_target, add_serving_assignment, find_remote_catalog_model_exact_blocking, local_process_payload, next_runtime_instance_id, plan_model_target_reconciliation, publish_runtime_llama_slots, publish_runtime_llama_unavailable, diff --git a/crates/mesh-llm-host-runtime/src/runtime/model_lifecycle/load.rs b/crates/mesh-llm-host-runtime/src/runtime/model_lifecycle/load.rs index 0ce0c4234c..bef090bc41 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/model_lifecycle/load.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/model_lifecycle/load.rs @@ -342,11 +342,14 @@ async fn finish_runtime_model_load( register_runtime_instance( ctx.runtime_instance_registry, ctx.node, - ctx.primary_model_name, - &loaded_name, - &instance_id, - Some(handle.context_length), - handle.capabilities, + RuntimeModelRegistration { + primary_model_name: ctx.primary_model_name, + model_name: &loaded_name, + instance_id: &instance_id, + context_length: Some(handle.context_length), + capabilities: handle.capabilities, + workload_class: handle.workload_class, + }, ) .await; ctx.node diff --git a/crates/mesh-llm-host-runtime/src/runtime/runtime_registry.rs b/crates/mesh-llm-host-runtime/src/runtime/runtime_registry.rs index 0e7aaeaca9..50c65c851b 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/runtime_registry.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/runtime_registry.rs @@ -15,6 +15,15 @@ use std::sync::Arc; pub(super) type RuntimeInstanceRegistry = Arc>>>>; +pub(super) struct RuntimeModelRegistration<'a> { + pub primary_model_name: &'a str, + pub model_name: &'a str, + pub instance_id: &'a str, + pub context_length: Option, + pub capabilities: models::ModelCapabilities, + pub workload_class: mesh::ModelWorkloadClass, +} + pub(super) fn next_runtime_instance_id(next_sequence: &mut u64) -> String { let instance_id = format!("runtime-{}", *next_sequence); *next_sequence = next_sequence.saturating_add(1); @@ -67,12 +76,16 @@ pub(super) fn reserve_runtime_capacity_for_model( pub(super) async fn register_runtime_instance( registry: &RuntimeInstanceRegistry, node: &mesh::Node, - primary_model_name: &str, - model_name: &str, - instance_id: &str, - context_length: Option, - capabilities: models::ModelCapabilities, + registration: RuntimeModelRegistration<'_>, ) { + let RuntimeModelRegistration { + primary_model_name, + model_name, + instance_id, + context_length, + capabilities, + workload_class, + } = registration; let (was_empty, context_changed, next_context) = { let mut guard = registry.lock().await; let instances = guard.entry(model_name.to_string()).or_default(); @@ -93,6 +106,7 @@ pub(super) async fn register_runtime_instance( primary_model_name, model_name, capabilities, + workload_class, ) .await; advertise_model_ready(node, primary_model_name, model_name, "").await; diff --git a/crates/mesh-llm-host-runtime/src/runtime/startup_handles.rs b/crates/mesh-llm-host-runtime/src/runtime/startup_handles.rs index 754a5040ac..9faa9f51a0 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/startup_handles.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/startup_handles.rs @@ -6,10 +6,10 @@ use super::{ DASHBOARD_CONTEXT_USAGE_REFRESH_INTERVAL, DashboardContextUsage, InitialPromptMode, InstanceLifecycleRecord, InstanceLifecycleState, LocalRuntimeModelHandle, LocalRuntimeModelStartSpec, OpenAiGuardrailPolicyHandle, RuntimeCapacityLedger, - RuntimeCapacityReservation, RuntimeInstanceRegistry, RuntimeOperationalEvent, - RuntimeResourcePlanningProfile, SPLIT_STANDBY_RETRY_INTERVAL, SplitCoordinatorAck, - SplitCoordinatorEvent, SplitRuntimeReason, SplitRuntimeStart, StartupPinnedGpuTarget, - StartupRuntimePlan, add_runtime_local_target, local_process_payload, + RuntimeCapacityReservation, RuntimeInstanceRegistry, RuntimeModelRegistration, + RuntimeOperationalEvent, RuntimeResourcePlanningProfile, SPLIT_STANDBY_RETRY_INTERVAL, + SplitCoordinatorAck, SplitCoordinatorEvent, SplitRuntimeReason, SplitRuntimeStart, + StartupPinnedGpuTarget, StartupRuntimePlan, add_runtime_local_target, local_process_payload, publish_runtime_llama_slots, publish_runtime_llama_unavailable, record_runtime_operational_event, record_runtime_operational_event_with_context, refresh_dashboard_context_usage, register_runtime_instance, remove_dashboard_context_usage, @@ -493,11 +493,14 @@ pub(super) async fn startup_register_loaded_runtime( register_runtime_instance( ctx.runtime_instance_registry, ctx.node, - ctx.primary_model_name, - loaded_name, - ctx.instance_id, - Some(handle.context_length), - handle.capabilities, + RuntimeModelRegistration { + primary_model_name: ctx.primary_model_name, + model_name: loaded_name, + instance_id: ctx.instance_id, + context_length: Some(handle.context_length), + capabilities: handle.capabilities, + workload_class: handle.workload_class, + }, ) .await; let payload = local_process_payload( diff --git a/crates/mesh-llm-host-runtime/src/runtime/tests/model_lifecycle.rs b/crates/mesh-llm-host-runtime/src/runtime/tests/model_lifecycle.rs index d5d98fafd3..f467ddcc69 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/tests/model_lifecycle.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/tests/model_lifecycle.rs @@ -386,21 +386,27 @@ async fn register_runtime_instance_preserves_existing_known_descriptor_capabilit register_runtime_instance( ®istry, &node, - vision_model, - vision_model, - "runtime-vision", - Some(8192), - vision_capabilities, + RuntimeModelRegistration { + primary_model_name: vision_model, + model_name: vision_model, + instance_id: "runtime-vision", + context_length: Some(8192), + capabilities: vision_capabilities, + workload_class: mesh::ModelWorkloadClass::CausalGeneration, + }, ) .await; register_runtime_instance( ®istry, &node, - vision_model, - text_model, - "runtime-text", - Some(8192), - models::ModelCapabilities::default(), + RuntimeModelRegistration { + primary_model_name: vision_model, + model_name: text_model, + instance_id: "runtime-text", + context_length: Some(8192), + capabilities: models::ModelCapabilities::default(), + workload_class: mesh::ModelWorkloadClass::Embedding, + }, ) .await; diff --git a/crates/mesh-llm-protocol/proto/node.proto b/crates/mesh-llm-protocol/proto/node.proto index d15c77e653..b20b94376d 100644 --- a/crates/mesh-llm-protocol/proto/node.proto +++ b/crates/mesh-llm-protocol/proto/node.proto @@ -261,6 +261,16 @@ message ServedModelMetadata { optional uint32 kv_head_count = 10; optional uint32 expert_count = 11; optional uint32 active_expert_count = 12; + optional ModelWorkloadClass workload_class = 13; +} + +enum ModelWorkloadClass { + MODEL_WORKLOAD_CLASS_UNSPECIFIED = 0; + MODEL_WORKLOAD_CLASS_CAUSAL_GENERATION = 1; + MODEL_WORKLOAD_CLASS_EMBEDDING = 2; + MODEL_WORKLOAD_CLASS_RERANK = 3; + MODEL_WORKLOAD_CLASS_ENCODER_DECODER = 4; + MODEL_WORKLOAD_CLASS_SPEECH_SYNTHESIS = 5; } message ServedModelIdentity { diff --git a/crates/mesh-llm-protocol/src/proto/node.rs b/crates/mesh-llm-protocol/src/proto/node.rs index 36b009cfeb..7c916df26a 100644 --- a/crates/mesh-llm-protocol/src/proto/node.rs +++ b/crates/mesh-llm-protocol/src/proto/node.rs @@ -321,6 +321,8 @@ pub struct ServedModelMetadata { pub expert_count: ::core::option::Option, #[prost(uint32, optional, tag = "12")] pub active_expert_count: ::core::option::Option, + #[prost(enumeration = "ModelWorkloadClass", optional, tag = "13")] + pub workload_class: ::core::option::Option, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ServedModelIdentity { @@ -985,6 +987,44 @@ impl ModelSourceKind { } } } +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ModelWorkloadClass { + Unspecified = 0, + CausalGeneration = 1, + Embedding = 2, + Rerank = 3, + EncoderDecoder = 4, + SpeechSynthesis = 5, +} +impl ModelWorkloadClass { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Unspecified => "MODEL_WORKLOAD_CLASS_UNSPECIFIED", + Self::CausalGeneration => "MODEL_WORKLOAD_CLASS_CAUSAL_GENERATION", + Self::Embedding => "MODEL_WORKLOAD_CLASS_EMBEDDING", + Self::Rerank => "MODEL_WORKLOAD_CLASS_RERANK", + Self::EncoderDecoder => "MODEL_WORKLOAD_CLASS_ENCODER_DECODER", + Self::SpeechSynthesis => "MODEL_WORKLOAD_CLASS_SPEECH_SYNTHESIS", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "MODEL_WORKLOAD_CLASS_UNSPECIFIED" => Some(Self::Unspecified), + "MODEL_WORKLOAD_CLASS_CAUSAL_GENERATION" => Some(Self::CausalGeneration), + "MODEL_WORKLOAD_CLASS_EMBEDDING" => Some(Self::Embedding), + "MODEL_WORKLOAD_CLASS_RERANK" => Some(Self::Rerank), + "MODEL_WORKLOAD_CLASS_ENCODER_DECODER" => Some(Self::EncoderDecoder), + "MODEL_WORKLOAD_CLASS_SPEECH_SYNTHESIS" => Some(Self::SpeechSynthesis), + _ => None, + } + } +} /// Shared enum #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] diff --git a/crates/mesh-llm-types/src/mesh/mod.rs b/crates/mesh-llm-types/src/mesh/mod.rs index 69d3d784d2..eb74c22b4c 100644 --- a/crates/mesh-llm-types/src/mesh/mod.rs +++ b/crates/mesh-llm-types/src/mesh/mod.rs @@ -63,8 +63,32 @@ fn is_false(value: &bool) -> bool { !*value } +/// The primary inference contract exposed by a loaded model runtime. +/// +/// This is deliberately separate from multimodal capabilities: a causal model +/// can accept image or audio inputs, while an embedding or reranking model has +/// a different output contract even when its input is text-only. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ModelWorkloadClass { + CausalGeneration, + Embedding, + Rerank, + EncoderDecoder, + SpeechSynthesis, + /// Explicit metadata from a newer peer that this node cannot interpret. + /// Unlike absent legacy metadata, this never authorizes inference. + #[serde(other)] + Unknown, +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] pub struct ServedModelMetadata { + /// `None` is the mixed-version representation for nodes that predate + /// workload advertisement. Callers must not treat it as evidence that a + /// non-chat endpoint is supported. + #[serde(skip_serializing_if = "Option::is_none")] + pub workload_class: Option, #[serde(skip_serializing_if = "Option::is_none")] pub architecture: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -92,8 +116,10 @@ pub struct ServedModelMetadata { } impl ServedModelMetadata { + /// Report whether all optional metadata is absent; a workload-only record is nonempty. pub fn is_empty(&self) -> bool { - self.architecture.is_none() + self.workload_class.is_none() + && self.architecture.is_none() && self.parameter_size.is_none() && self.parameter_count_b.is_none() && self.quant.is_none() diff --git a/crates/mesh-native-serving-plugin-host/src/test_support.rs b/crates/mesh-native-serving-plugin-host/src/test_support.rs index 9ca77a5966..d145656bf1 100644 --- a/crates/mesh-native-serving-plugin-host/src/test_support.rs +++ b/crates/mesh-native-serving-plugin-host/src/test_support.rs @@ -243,12 +243,14 @@ unsafe extern "C" fn fake_discard_proposal( event: *const abi::ProposalDiscard, ) -> abi::PluginStatus { let state = unsafe { &*instance.cast::() }; - state.events.lock().unwrap().push("discard"); state .discard_reasons .lock() .unwrap() .push(unsafe { (*event).reason }); + // Publish the event only after its observation is complete: tests wait on + // this marker before reading the discard reason from another thread. + state.events.lock().unwrap().push("discard"); thread::sleep(state.report_delay); abi::PluginStatus::OK } diff --git a/crates/openai-frontend/Cargo.toml b/crates/openai-frontend/Cargo.toml index 70ca397a78..619bc0a0d5 100644 --- a/crates/openai-frontend/Cargo.toml +++ b/crates/openai-frontend/Cargo.toml @@ -9,7 +9,8 @@ homepage = "https://github.com/Mesh-LLM/mesh-llm" [dependencies] async-trait = "0.1" -axum = "0.8" +axum = { version = "0.8", features = ["multipart"] } +base64 = "0.22" futures-core = "0.3" futures-util = "0.3" mesh-llm-guardrails = { path = "../mesh-llm-guardrails", version = "0.76.1" } diff --git a/crates/openai-frontend/README.md b/crates/openai-frontend/README.md index 733d36a316..855c22401c 100644 --- a/crates/openai-frontend/README.md +++ b/crates/openai-frontend/README.md @@ -6,8 +6,8 @@ entry points. This crate owns the public API shapes and route machinery that should not be duplicated inside `skippy-server`. Stage server code should provide a thin backend adapter that implements the frontend trait, while this crate handles -request/response JSON, OpenAI-style errors, `/v1/models`, chat completions, and -streaming Server-Sent Events framing. Mesh uses this as the single OpenAI +request/response JSON, OpenAI-style errors, model discovery, generation, +embedding, rerank, audio, and streaming Server-Sent Events framing. Mesh uses this as the single OpenAI surface for embedded single-stage and stage-split serving. Mesh-local compatibility wrappers should stay thin. Request normalization, @@ -31,6 +31,11 @@ For the concrete benchy command and contract, see | `POST /v1/chat/completions` | Supported | Handles streaming and non-streaming response shapes. | | `POST /v1/completions` | Supported | Handles streaming and non-streaming response shapes. | | `POST /v1/responses` | Supported | Adapts OpenAI responses requests onto chat/completion backend calls and preserves response metadata where possible. | +| `POST /v1/embeddings` | Supported | String and token inputs, float or base64 vectors, usage accounting. Runtime support is model-gated. | +| `POST /v1/rerank` | Supported | Cross-encoder query/document scoring with optional document return. Runtime support is model-gated. | +| `POST /v1/audio/speech` | Supported | Binary response with format-specific content type. The native backend currently produces WAV or PCM and accepts only `voice: "default"`; speaker selection fails with a structured unsupported error. | +| `POST /v1/audio/transcriptions` | Supported | Bounded multipart audio upload with JSON or text response. Runtime support is model-gated. | +| `POST /v1/audio/translations` | Supported | Bounded multipart audio upload translated to English, with JSON or text response. Runtime support is model-gated. | | `GET /health` / `GET /healthz` | Supported | Lightweight liveness probes for hosts and CI smoke tests. | | `GET /readyz` | Supported | Backend readiness probe that verifies model discovery through `OpenAiBackend::models`. | | Server-Sent Events | Supported | Emits OpenAI-style JSON chunks and `[DONE]`. | @@ -52,13 +57,14 @@ For the concrete benchy command and contract, see | Client nonce | Supported | Accepts `x-capsule-client-nonce` only when it is exactly one valid UUIDv4; a missing, invalid, non-UUIDv4, or duplicated value is replaced with a freshly minted UUIDv4. When this frontend mints the value it stamps `x-capsule-nonce-origin: frontend`; a forwarded (client-supplied) nonce carries no origin marker, and any inbound `x-capsule-nonce-origin` is always stripped so a caller cannot forge it. Both headers are echoed on covered responses: the axum router and, via the host runtime's forwarding rebuild, the public `:9337` proxy JSON/SSE paths (including the pipeline/MoA strong-model path and remapped upstream error responses). Locally synthesized error responses (e.g. no-target `503`s and `/v1/models` listings) do not yet carry the headers; threading the request nonce onto those senders is a cross-cutting signature change tracked as a follow-up. The origin marker asserts only that *this* frontend minted the value, not that it is the original ingress for a remote-routed request. | | Backend timeout | Supported | Configurable via `OpenAiFrontendConfig` or the `MESH_OPENAI_BACKEND_TIMEOUT_SECS` environment variable; defaults to 600 seconds (`0` disables it) and maps timeouts to OpenAI-shaped 504 errors. | | Agent session header | Supported | Set `MESH_AGENT_SESSION_HEADER` to accept a trusted upstream header as the stable agent-session identity. | -| embeddings/rerank/infill/audio/vision | Out of scope | Not needed for staged text benchmark entrypoints. | +| Vision input | Supported | Preserved through chat/Responses content parts and executed by projector-backed runtimes. | +| Non-chat staging | Fail closed | Embedding, rerank, encoder-decoder, and speech-synthesis models currently require an unsplit full-model runtime. | ## Shape ```mermaid flowchart TB - C["OpenAI-compatible client
chat, completions, responses"] --> R["openai-frontend
Axum routes"] + C["OpenAI-compatible client
generation, embeddings, audio"] --> R["openai-frontend
Axum routes"] R --> Parse["request parsing
validation
normalization
OpenAI errors"] Parse --> B["OpenAiBackend implementation"] B --> Local["embedded single-stage
skippy runtime"] @@ -69,7 +75,10 @@ flowchart TB R --> C ``` -The backend boundary is intentionally small: +The backend boundary below is a partial generation example. The complete +[`OpenAiBackend` trait](src/backend.rs) also defines `embeddings`, `rerank`, +`audio_speech`, `audio_transcription`, and `audio_translation`; override their +default unsupported responses to serve the corresponding non-chat endpoints. ```rust #[async_trait] @@ -78,6 +87,7 @@ pub trait OpenAiBackend { async fn chat_completion( &self, request: ChatCompletionRequest, + context: OpenAiRequestContext, ) -> OpenAiResult; async fn chat_completion_stream( &self, @@ -87,6 +97,7 @@ pub trait OpenAiBackend { async fn completion( &self, request: CompletionRequest, + context: OpenAiRequestContext, ) -> OpenAiResult; async fn completion_stream( &self, @@ -116,6 +127,9 @@ chain remains backend-owned. | `/v1/chat/completions` | Parse common and advanced fields, stream/non-stream envelopes | Tokenization, sampling, stop handling, usage, feature execution | Supported with backend feature guards | | `/v1/completions` | Prompt parsing and response envelopes | Token prompts, sampling, stop handling, usage | Supported with backend feature guards | | `/v1/responses` | Translate request/response shapes onto the backend contract | Execute the resulting chat/completion request | Supported | +| `/v1/embeddings` | Parse OpenAI input/encoding shapes and serialize vectors | Tokenize or accept token IDs, pool, normalize, and report usage | Supported for compatible local full models | +| `/v1/rerank` | Parse query/documents and serialize ranked results | Execute classifier scoring and report usage | Supported for compatible local full models | +| `/v1/audio/*` | Parse JSON or bounded multipart bodies; return JSON, text, or binary media | Execute TTS or projector-backed speech recognition | Supported for compatible local full models | | HTTP operations | Health/readiness, fallbacks, payload limits, content-type handling | Model readiness and backend timeouts | Supported | | Streaming | SSE chunks, `[DONE]`, cancellation context | Produce deltas, usage, and optional logprob/tool metadata | Supported with backend feature guards | | Chat templates | Preserve OpenAI message shape | Apply model-aware chat templates through the skippy ABI | Backend-owned | @@ -130,7 +144,7 @@ chain remains backend-owned. | Logprobs | Parse and preserve request/response shape | Expose logits/probabilities | Frontend ready; backend-gated | | Tools/function calling | Parse and preserve tool schemas and tool-call response shape | Generate tool calls | Frontend ready; backend-gated | | JSON schema/grammar | Parse and preserve `response_format` | Constrained decoding | Frontend ready; backend-gated | -| Embeddings/rerank/infill/audio | Route fallback/error handling | Runtime implementation if reintroduced | Out of current scope | +| Non-chat workload gates | Preserve request/response contracts and structured errors | Probe the native model class and reject unsupported stage shapes | Supported | | Metrics | Request IDs and tracing context | Stage/OpenAI telemetry emitted to `metrics-server` | Supported | ## Stage-Server Integration @@ -144,3 +158,6 @@ small adapter: That keeps `serve-openai` and the embedded mesh path thin: parse or build the runtime config, construct the backend, pass it to `openai_frontend::router`, and serve the Axum app. + +See [`docs/NON_CHAT_MODELS.md`](../../docs/NON_CHAT_MODELS.md) for endpoint +examples, execution boundaries, and certification policy. diff --git a/crates/openai-frontend/src/audio.rs b/crates/openai-frontend/src/audio.rs new file mode 100644 index 0000000000..4e492fce4f --- /dev/null +++ b/crates/openai-frontend/src/audio.rs @@ -0,0 +1,164 @@ +use serde::{Deserialize, Serialize}; + +use crate::{OpenAiError, OpenAiResult}; + +const MAX_AUDIO_BYTES: usize = 64 * 1024 * 1024; + +#[derive(Debug, Clone, Copy, Default, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum AudioFormat { + #[default] + Mp3, + Opus, + Aac, + Flac, + Wav, + Pcm, +} + +impl AudioFormat { + /// Return the media type corresponding to the requested audio wire encoding. + pub const fn content_type(self) -> &'static str { + match self { + Self::Mp3 => "audio/mpeg", + Self::Opus => "audio/opus", + Self::Aac => "audio/aac", + Self::Flac => "audio/flac", + Self::Wav => "audio/wav", + Self::Pcm => "audio/pcm", + } + } +} + +#[derive(Debug, Clone, Deserialize, PartialEq)] +pub struct AudioSpeechRequest { + pub model: String, + pub input: String, + pub voice: String, + #[serde(default)] + pub response_format: AudioFormat, + #[serde(default = "default_speed")] + pub speed: f32, +} + +/// Omitted speed retains the model's unscaled playback rate. +fn default_speed() -> f32 { + 1.0 +} + +impl AudioSpeechRequest { + /// Validate required inputs and the finite OpenAI playback-speed range. + pub fn validate(&self) -> OpenAiResult<()> { + if self.model.trim().is_empty() || self.input.is_empty() || self.voice.trim().is_empty() { + return Err(OpenAiError::invalid_request( + "model, input, and voice must not be empty", + )); + } + if !(0.25..=4.0).contains(&self.speed) || !self.speed.is_finite() { + return Err(OpenAiError::invalid_request( + "speed must be a finite value between 0.25 and 4.0", + )); + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AudioResponse { + pub bytes: Vec, + pub content_type: String, +} + +impl AudioResponse { + /// Reject empty backend output before constructing a binary HTTP response. + pub fn new(bytes: Vec, content_type: impl Into) -> OpenAiResult { + if bytes.is_empty() { + return Err(OpenAiError::backend( + "audio backend returned an empty payload", + )); + } + Ok(Self { + bytes, + content_type: content_type.into(), + }) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct AudioTranscriptionRequest { + pub model: String, + pub file: Vec, + pub filename: Option, + pub language: Option, + pub prompt: Option, + pub response_format: String, + pub temperature: Option, +} + +impl AudioTranscriptionRequest { + pub const MAX_FILE_BYTES: usize = MAX_AUDIO_BYTES; + + /// Enforce upload size, output format, and the finite `[0, 1]` temperature range. + pub fn validate(&self) -> OpenAiResult<()> { + if self.model.trim().is_empty() { + return Err(OpenAiError::invalid_request("model must not be empty")); + } + if self.file.is_empty() { + return Err(OpenAiError::invalid_request("audio file must not be empty")); + } + if self.file.len() > Self::MAX_FILE_BYTES { + return Err(OpenAiError::payload_too_large(format!( + "audio file exceeds the {} byte limit", + Self::MAX_FILE_BYTES + ))); + } + if !matches!(self.response_format.as_str(), "json" | "text") { + return Err(OpenAiError::unsupported( + "response_format must be 'json' or 'text'", + )); + } + if self + .temperature + .is_some_and(|value| !value.is_finite() || !(0.0..=1.0).contains(&value)) + { + return Err(OpenAiError::invalid_request( + "temperature must be a finite value between 0.0 and 1.0", + )); + } + Ok(()) + } +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct AudioTranscriptionResponse { + pub text: String, +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Build a valid transcription fixture for multipart parameter tests. + fn transcription(temperature: Option) -> AudioTranscriptionRequest { + AudioTranscriptionRequest { + model: "fixture".to_string(), + file: vec![1], + filename: None, + language: None, + prompt: None, + response_format: "json".to_string(), + temperature, + } + } + + #[test] + /// Reject non-finite and out-of-range transcription temperatures. + fn transcription_temperature_is_bounded_to_openai_range() { + for temperature in [None, Some(0.0), Some(0.5), Some(1.0)] { + assert!(transcription(temperature).validate().is_ok()); + } + for temperature in [Some(-0.1), Some(1.1), Some(f32::NAN), Some(f32::INFINITY)] { + assert!(transcription(temperature).validate().is_err()); + } + } +} diff --git a/crates/openai-frontend/src/backend.rs b/crates/openai-frontend/src/backend.rs index e6aae05d03..f70e470ff9 100644 --- a/crates/openai-frontend/src/backend.rs +++ b/crates/openai-frontend/src/backend.rs @@ -11,11 +11,16 @@ use futures_core::Stream; use tokio::sync::Notify; use crate::{ + audio::{ + AudioResponse, AudioSpeechRequest, AudioTranscriptionRequest, AudioTranscriptionResponse, + }, chat::{ChatCompletionChunk, ChatCompletionRequest, ChatCompletionResponse}, completions::{CompletionChunk, CompletionRequest, CompletionResponse}, + embeddings::{EmbeddingResponse, EmbeddingsRequest}, errors::OpenAiError, lifecycle::RequestId, models::ModelObject, + rerank::{RerankRequest, RerankResponse}, }; pub type ChatCompletionStream = @@ -184,6 +189,61 @@ pub trait OpenAiBackend: Send + Sync + 'static { "/v1/completions streaming is not supported by this backend", )) } + + /// Embed text or token batches. The default rejects unsupported backends. + async fn embeddings( + &self, + _request: EmbeddingsRequest, + _context: OpenAiRequestContext, + ) -> OpenAiResult { + Err(OpenAiError::unsupported( + "/v1/embeddings is not supported by this backend", + )) + } + + /// Rank documents against a query, preserving their original indexes. + async fn rerank( + &self, + _request: RerankRequest, + _context: OpenAiRequestContext, + ) -> OpenAiResult { + Err(OpenAiError::unsupported( + "/v1/rerank is not supported by this backend", + )) + } + + /// Synthesize binary audio; unsupported codecs and voices must return errors. + async fn audio_speech( + &self, + _request: AudioSpeechRequest, + _context: OpenAiRequestContext, + ) -> OpenAiResult { + Err(OpenAiError::unsupported( + "/v1/audio/speech is not supported by this backend", + )) + } + + /// Transcribe uploaded audio in its source language using the request context. + async fn audio_transcription( + &self, + _request: AudioTranscriptionRequest, + _context: OpenAiRequestContext, + ) -> OpenAiResult { + Err(OpenAiError::unsupported( + "/v1/audio/transcriptions is not supported by this backend", + )) + } + + /// Translate uploaded audio into English; unsupported backends fail explicitly. + async fn audio_translation( + &self, + _request: AudioTranscriptionRequest, + _context: OpenAiRequestContext, + ) -> OpenAiResult { + Err(OpenAiError::unsupported( + "/v1/audio/translations is not supported by this backend", + )) + } } pub(crate) type SharedBackend = Arc; diff --git a/crates/openai-frontend/src/chat.rs b/crates/openai-frontend/src/chat.rs index a6ac07f00f..f1babd0732 100644 --- a/crates/openai-frontend/src/chat.rs +++ b/crates/openai-frontend/src/chat.rs @@ -141,10 +141,10 @@ fn validate_tools_value(value: &Value) -> Result<(), String> { let Some(function) = tool.get("function").and_then(Value::as_object) else { return Err(format!("tools[{index}].function must be an object")); }; - if !function + if function .get("name") .and_then(Value::as_str) - .is_some_and(|name| !name.trim().is_empty()) + .is_none_or(|name| name.trim().is_empty()) { return Err(format!( "tools[{index}].function.name must be a non-empty string" diff --git a/crates/openai-frontend/src/embeddings.rs b/crates/openai-frontend/src/embeddings.rs new file mode 100644 index 0000000000..f163817760 --- /dev/null +++ b/crates/openai-frontend/src/embeddings.rs @@ -0,0 +1,209 @@ +use base64::{Engine, engine::general_purpose::STANDARD}; +use serde::{Deserialize, Serialize}; + +use crate::{OpenAiError, OpenAiResult, Usage}; + +const DEFAULT_ENCODING_FORMAT: &str = "float"; + +/// OpenAI-compatible embedding input. Token arrays are preserved so clients +/// can avoid a second tokenizer pass when they already own tokenization. +#[derive(Debug, Clone, Deserialize, PartialEq)] +#[serde(untagged)] +pub enum EmbeddingInput { + Text(String), + Texts(Vec), + Tokens(Vec), + TokenArrays(Vec>), +} + +impl EmbeddingInput { + /// Count independent inputs; a single token array is one input, not a batch. + pub fn len(&self) -> usize { + match self { + Self::Text(_) | Self::Tokens(_) => 1, + Self::Texts(values) => values.len(), + Self::TokenArrays(values) => values.len(), + } + } + + /// Reject empty batches and empty members in text or pre-tokenized input. + pub fn is_empty(&self) -> bool { + match self { + Self::Text(value) => value.is_empty(), + Self::Texts(values) => values.is_empty() || values.iter().any(String::is_empty), + Self::Tokens(values) => values.is_empty(), + Self::TokenArrays(values) => values.is_empty() || values.iter().any(Vec::is_empty), + } + } + + /// Detect negative token IDs without applying tokenizer rules to text input. + fn contains_invalid_token(&self) -> bool { + match self { + Self::Text(_) | Self::Texts(_) => false, + Self::Tokens(tokens) => tokens.iter().any(|token| *token < 0), + Self::TokenArrays(inputs) => inputs.iter().flatten().any(|token| *token < 0), + } + } +} + +#[derive(Debug, Clone, Deserialize, PartialEq)] +pub struct EmbeddingsRequest { + pub model: String, + pub input: EmbeddingInput, + #[serde(default = "default_encoding_format")] + pub encoding_format: String, + #[serde(default)] + pub dimensions: Option, + #[serde(default)] + pub user: Option, +} + +/// Preserve the endpoint's default numeric-vector response representation. +fn default_encoding_format() -> String { + DEFAULT_ENCODING_FORMAT.to_string() +} + +impl EmbeddingsRequest { + /// Validate batch content, output encoding, and optional positive dimensions. + pub fn validate(&self) -> OpenAiResult<()> { + if self.model.trim().is_empty() { + return Err(OpenAiError::invalid_request("model must not be empty")); + } + if self.input.is_empty() { + return Err(OpenAiError::invalid_request( + "embedding input must contain at least one non-empty item", + )); + } + if self.input.contains_invalid_token() { + return Err(OpenAiError::invalid_request( + "embedding token IDs must be non-negative", + )); + } + if !matches!(self.encoding_format.as_str(), "float" | "base64") { + return Err(OpenAiError::invalid_request( + "encoding_format must be 'float' or 'base64'", + )); + } + if self.dimensions == Some(0) { + return Err(OpenAiError::invalid_request( + "dimensions must be greater than zero", + )); + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Embedding { + pub values: Vec, + pub index: usize, +} + +#[derive(Debug, Clone, Serialize, PartialEq)] +#[serde(untagged)] +pub enum EmbeddingOutput { + Float(Vec), + Base64(String), +} + +#[derive(Debug, Clone, Serialize, PartialEq)] +pub struct EmbeddingObject { + pub object: &'static str, + pub embedding: EmbeddingOutput, + pub index: usize, +} + +#[derive(Debug, Clone, Serialize, PartialEq)] +pub struct EmbeddingResponse { + pub object: &'static str, + pub data: Vec, + pub model: String, + pub usage: Usage, +} + +impl EmbeddingResponse { + /// Preserve input indexes and encode vectors with the already-validated format. + /// Embedding usage contains prompt tokens only; this endpoint generates none. + pub fn from_embeddings( + model: String, + embeddings: Vec, + prompt_tokens: u32, + encoding_format: &str, + ) -> Self { + let data = embeddings + .into_iter() + .map(|embedding| EmbeddingObject { + object: "embedding", + embedding: if encoding_format == "base64" { + EmbeddingOutput::Base64(encode_f32_base64(&embedding.values)) + } else { + EmbeddingOutput::Float(embedding.values) + }, + index: embedding.index, + }) + .collect(); + Self { + object: "list", + data, + model, + usage: Usage { + prompt_tokens, + completion_tokens: 0, + total_tokens: prompt_tokens, + ..Usage::default() + }, + } + } +} + +/// Encode little-endian IEEE-754 samples, independent of the host byte order. +fn encode_f32_base64(values: &[f32]) -> String { + let mut bytes = Vec::with_capacity(std::mem::size_of_val(values)); + for value in values { + bytes.extend_from_slice(&value.to_le_bytes()); + } + STANDARD.encode(bytes) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + /// Verify embeddings use little-endian float32 base64 encoding. + fn base64_encoding_is_little_endian_f32() { + assert_eq!(encode_f32_base64(&[1.0, -2.0]), "AACAPwAAAMA="); + } + + #[test] + /// Reject embedding requests without inputs or with unsupported encodings. + fn request_rejects_empty_batches_and_unknown_formats() { + let mut request = EmbeddingsRequest { + model: "embed".into(), + input: EmbeddingInput::Texts(Vec::new()), + encoding_format: "float".into(), + dimensions: None, + user: None, + }; + assert!(request.validate().is_err()); + request.input = EmbeddingInput::Text("hello".into()); + request.encoding_format = "hex".into(); + assert!(request.validate().is_err()); + } + + #[test] + /// Reject negative embedding token IDs before backend execution. + fn request_rejects_negative_token_ids() { + let request = EmbeddingsRequest { + model: "embed".into(), + input: EmbeddingInput::TokenArrays(vec![vec![1, 2], vec![3, -1]]), + encoding_format: "float".into(), + dimensions: None, + user: None, + }; + + let error = request.validate().expect_err("negative IDs must fail"); + assert_eq!(error.status(), axum::http::StatusCode::BAD_REQUEST); + assert!(error.body().error.message.contains("non-negative")); + } +} diff --git a/crates/openai-frontend/src/guardrails/compact.rs b/crates/openai-frontend/src/guardrails/compact.rs index 57c8b337bc..fc28c94602 100644 --- a/crates/openai-frontend/src/guardrails/compact.rs +++ b/crates/openai-frontend/src/guardrails/compact.rs @@ -6,13 +6,18 @@ use mesh_llm_guardrails::{ }; use crate::{ + audio::{ + AudioResponse, AudioSpeechRequest, AudioTranscriptionRequest, AudioTranscriptionResponse, + }, backend::{ ChatCompletionStream, CompletionStream, OpenAiBackend, OpenAiRequestContext, OpenAiResult, }, chat::{ChatCompletionRequest, ChatCompletionResponse}, completions::{CompletionRequest, CompletionResponse}, + embeddings::{EmbeddingResponse, EmbeddingsRequest}, errors::OpenAiError, models::ModelObject, + rerank::{RerankRequest, RerankResponse}, }; pub struct CompactingOpenAiBackend { @@ -108,4 +113,49 @@ impl OpenAiBackend for CompactingOpenAiBackend { ) -> OpenAiResult { self.backend.completion_stream(request, context).await } + + /// Forward embeddings and request context without chat processing. + async fn embeddings( + &self, + request: EmbeddingsRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.embeddings(request, context).await + } + + /// Forward reranking and request context without chat processing. + async fn rerank( + &self, + request: RerankRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.rerank(request, context).await + } + + /// Forward speech generation and request context unchanged. + async fn audio_speech( + &self, + request: AudioSpeechRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.audio_speech(request, context).await + } + + /// Forward multipart transcription and request context unchanged. + async fn audio_transcription( + &self, + request: AudioTranscriptionRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.audio_transcription(request, context).await + } + + /// Forward multipart translation and request context unchanged. + async fn audio_translation( + &self, + request: AudioTranscriptionRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.audio_translation(request, context).await + } } diff --git a/crates/openai-frontend/src/guardrails/mod.rs b/crates/openai-frontend/src/guardrails/mod.rs index 09a4924fc4..f4fea73c7d 100644 --- a/crates/openai-frontend/src/guardrails/mod.rs +++ b/crates/openai-frontend/src/guardrails/mod.rs @@ -3,12 +3,17 @@ use std::sync::Arc; use async_trait::async_trait; use crate::{ + audio::{ + AudioResponse, AudioSpeechRequest, AudioTranscriptionRequest, AudioTranscriptionResponse, + }, backend::{ ChatCompletionStream, CompletionStream, OpenAiBackend, OpenAiRequestContext, OpenAiResult, }, chat::{ChatCompletionRequest, ChatCompletionResponse}, completions::{CompletionRequest, CompletionResponse}, + embeddings::{EmbeddingResponse, EmbeddingsRequest}, models::ModelObject, + rerank::{RerankRequest, RerankResponse}, }; mod compact; @@ -350,6 +355,51 @@ impl OpenAiBackend for GuardedOpenAiBackend { ) -> OpenAiResult { self.backend.completion_stream(request, context).await } + + /// Forward embeddings and request context without chat processing. + async fn embeddings( + &self, + request: EmbeddingsRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.embeddings(request, context).await + } + + /// Forward reranking and request context without chat processing. + async fn rerank( + &self, + request: RerankRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.rerank(request, context).await + } + + /// Forward speech generation and request context unchanged. + async fn audio_speech( + &self, + request: AudioSpeechRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.audio_speech(request, context).await + } + + /// Forward multipart transcription and request context unchanged. + async fn audio_transcription( + &self, + request: AudioTranscriptionRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.audio_transcription(request, context).await + } + + /// Forward multipart translation and request context unchanged. + async fn audio_translation( + &self, + request: AudioTranscriptionRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.audio_translation(request, context).await + } } #[cfg(test)] diff --git a/crates/openai-frontend/src/hooks.rs b/crates/openai-frontend/src/hooks.rs index b78914f28b..5d227008c0 100644 --- a/crates/openai-frontend/src/hooks.rs +++ b/crates/openai-frontend/src/hooks.rs @@ -4,6 +4,9 @@ use async_trait::async_trait; use serde_json::Value; use crate::{ + audio::{ + AudioResponse, AudioSpeechRequest, AudioTranscriptionRequest, AudioTranscriptionResponse, + }, backend::{ ChatCompletionStream, CompletionStream, OpenAiBackend, OpenAiRequestContext, OpenAiResult, }, @@ -12,7 +15,9 @@ use crate::{ ChatMessage, MessageContent, MessageContentPart, capsule_id_is_valid, }, completions::{CompletionRequest, CompletionResponse}, + embeddings::{EmbeddingResponse, EmbeddingsRequest}, models::ModelObject, + rerank::{RerankRequest, RerankResponse}, }; pub const MESH_HOOKS_FIELD: &str = "mesh_hooks"; @@ -377,7 +382,7 @@ impl Drop for TerminalGuard { /// - A chunk carrying an error fires [`ChatCompletionOutcome::Error`] /// immediately — matching the non-streaming path, which never waits for a /// graceful end once the backend has already reported failure. -/// - Both fire via [`TerminalGuard::fire_detached`], since neither can +/// - Both fire via `TerminalGuard::fire_detached`, since neither can /// `.await` inside `poll_next`. /// - Dropping this wrapper before either of the above happens — an outer /// timeout, or the client disconnecting mid-stream — drops the @@ -609,6 +614,51 @@ impl OpenAiBackend for HookedOpenAiBackend { ) -> OpenAiResult { self.backend.completion_stream(request, context).await } + + /// Forward embeddings and request context without chat processing. + async fn embeddings( + &self, + request: EmbeddingsRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.embeddings(request, context).await + } + + /// Forward reranking and request context without chat processing. + async fn rerank( + &self, + request: RerankRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.rerank(request, context).await + } + + /// Forward speech generation and request context unchanged. + async fn audio_speech( + &self, + request: AudioSpeechRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.audio_speech(request, context).await + } + + /// Forward multipart transcription and request context unchanged. + async fn audio_transcription( + &self, + request: AudioTranscriptionRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.audio_transcription(request, context).await + } + + /// Forward multipart translation and request context unchanged. + async fn audio_translation( + &self, + request: AudioTranscriptionRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.backend.audio_translation(request, context).await + } } pub fn chat_mesh_hooks_enabled(request: &ChatCompletionRequest) -> bool { diff --git a/crates/openai-frontend/src/lib.rs b/crates/openai-frontend/src/lib.rs index a6a032080e..6f88ef3eb5 100644 --- a/crates/openai-frontend/src/lib.rs +++ b/crates/openai-frontend/src/lib.rs @@ -1,19 +1,26 @@ +pub mod audio; pub mod backend; mod backend_lifecycle; pub mod chat; pub mod common; pub mod completions; +pub mod embeddings; pub mod errors; mod guardrails; pub mod hooks; pub mod lifecycle; pub mod models; mod request_lifecycle; +pub mod rerank; pub mod responses; pub mod router; pub mod sse; mod stream_lifecycle; +pub use audio::{ + AudioFormat, AudioResponse, AudioSpeechRequest, AudioTranscriptionRequest, + AudioTranscriptionResponse, +}; pub use backend::{ CancellationToken, ChatCompletionStream, CompletionStream, OpenAiBackend, OpenAiRequestContext, OpenAiResult, @@ -34,6 +41,9 @@ pub use completions::{ CompletionChoice, CompletionChunk, CompletionChunkChoice, CompletionPrompt, CompletionRequest, CompletionResponse, }; +pub use embeddings::{ + Embedding, EmbeddingInput, EmbeddingOutput, EmbeddingResponse, EmbeddingsRequest, +}; pub use errors::{OpenAiError, OpenAiErrorKind, already_openai_error, map_upstream_error_body}; pub use guardrails::{ CompactingOpenAiBackend, CompactionConfig, CompactionDecision, CompactionOverride, @@ -56,6 +66,7 @@ pub use lifecycle::{ parse_single_request_id, request_id_from_headers_or_generate, request_id_response_header, }; pub use models::{ModelId, ModelIdError, ModelObject, ModelsResponse}; +pub use rerank::{RerankDocument, RerankRequest, RerankResponse, RerankResult}; pub use responses::{ NormalizationOutcome, ResponseAdapterMode, ResponsesRequest, StreamUsage, chat_usage_to_responses_usage, normalize_openai_compat_request, parse_chat_stream_chunk, diff --git a/crates/openai-frontend/src/lifecycle.rs b/crates/openai-frontend/src/lifecycle.rs index 47159f910f..b072531e77 100644 --- a/crates/openai-frontend/src/lifecycle.rs +++ b/crates/openai-frontend/src/lifecycle.rs @@ -73,6 +73,11 @@ pub enum OpenAiFrontendRoute { Healthz, Readyz, Models, + Embeddings, + Rerank, + AudioSpeech, + AudioTranscriptions, + AudioTranslations, ChatCompletions, Completions, Responses, @@ -83,6 +88,11 @@ pub enum OpenAiFrontendRoute { #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum OpenAiBackendOperation { Models, + Embeddings, + Rerank, + AudioSpeech, + AudioTranscription, + AudioTranslation, ChatCompletion, ChatCompletionStream, Completion, diff --git a/crates/openai-frontend/src/rerank.rs b/crates/openai-frontend/src/rerank.rs new file mode 100644 index 0000000000..65d0024414 --- /dev/null +++ b/crates/openai-frontend/src/rerank.rs @@ -0,0 +1,95 @@ +use serde::{Deserialize, Serialize}; + +use crate::{OpenAiError, OpenAiResult, Usage}; + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] +#[serde(untagged)] +pub enum RerankDocument { + Text(String), + Object(serde_json::Value), +} + +impl RerankDocument { + /// Borrow plain text or a document object's required string `text` field. + pub fn text(&self) -> OpenAiResult<&str> { + match self { + Self::Text(text) => Ok(text), + Self::Object(value) => value + .get("text") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| { + OpenAiError::invalid_request( + "rerank document objects must contain a string 'text' field", + ) + }), + } + } +} + +#[derive(Debug, Clone, Deserialize, PartialEq)] +pub struct RerankRequest { + pub model: String, + pub query: String, + pub documents: Vec, + #[serde(default)] + pub top_n: Option, + #[serde(default)] + pub return_documents: bool, +} + +impl RerankRequest { + /// Validate every document before admission, including optional top-N bounds. + pub fn validate(&self) -> OpenAiResult<()> { + if self.model.trim().is_empty() { + return Err(OpenAiError::invalid_request("model must not be empty")); + } + if self.query.is_empty() { + return Err(OpenAiError::invalid_request("query must not be empty")); + } + if self.documents.is_empty() { + return Err(OpenAiError::invalid_request( + "documents must contain at least one item", + )); + } + for document in &self.documents { + if document.text()?.is_empty() { + return Err(OpenAiError::invalid_request( + "rerank documents must not be empty", + )); + } + } + if self.top_n == Some(0) { + return Err(OpenAiError::invalid_request( + "top_n must be greater than zero", + )); + } + Ok(()) + } +} + +#[derive(Debug, Clone, Serialize, PartialEq)] +pub struct RerankResult { + pub index: usize, + pub relevance_score: f32, + #[serde(skip_serializing_if = "Option::is_none")] + pub document: Option, +} + +#[derive(Debug, Clone, Serialize, PartialEq)] +pub struct RerankResponse { + pub id: String, + pub results: Vec, + pub usage: Usage, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + /// Require textual content in rerank document objects. + fn document_objects_require_text() { + let document = RerankDocument::Object(serde_json::json!({"title": "missing"})); + assert!(document.text().is_err()); + } +} diff --git a/crates/openai-frontend/src/router.rs b/crates/openai-frontend/src/router.rs index 56846b40e3..18d2c128ed 100644 --- a/crates/openai-frontend/src/router.rs +++ b/crates/openai-frontend/src/router.rs @@ -7,8 +7,15 @@ use std::{ use axum::{ Json, Router, body::Body, - extract::{DefaultBodyLimit, Extension, State, rejection::JsonRejection}, - http::{HeaderMap, HeaderValue, Method, Request, StatusCode, Uri, header::HeaderName}, + extract::{ + DefaultBodyLimit, Extension, Multipart, State, + multipart::{MultipartError, MultipartRejection}, + rejection::JsonRejection, + }, + http::{ + HeaderMap, HeaderValue, Method, Request, StatusCode, Uri, + header::{self, HeaderName}, + }, middleware::{self, Next}, response::{IntoResponse, Response, sse::Event}, routing::{get, post}, @@ -19,11 +26,15 @@ use serde::Serialize; use serde_json::Value; use crate::{ + audio::{ + AudioResponse, AudioSpeechRequest, AudioTranscriptionRequest, AudioTranscriptionResponse, + }, backend::{OpenAiBackend, OpenAiRequestContext, OpenAiResult, SharedBackend}, backend_lifecycle::{call_backend, call_backend_with_context}, chat::{CapsuleMarker, ChatCompletionChunk, ChatCompletionRequest}, common::{AgentSessionIdentity, AgentSessionSource, Usage}, completions::CompletionRequest, + embeddings::{EmbeddingResponse, EmbeddingsRequest}, errors::OpenAiError, lifecycle::{ CLIENT_NONCE_HEADER, CLIENT_NONCE_ORIGIN_HEADER, OpenAiBackendOperation, @@ -33,6 +44,7 @@ use crate::{ }, models::ModelsResponse, request_lifecycle::RequestLifecycle, + rerank::{RerankRequest, RerankResponse}, responses::{ ResponseAdapterMode, ResponseSseState, chunk_delta_text, normalize_openai_compat_request, responses_stream_completed_event_with_sequence, responses_stream_content_part_added_event, @@ -50,6 +62,7 @@ use crate::{ const AGENT_SESSION_HEADER_ENV: &str = "MESH_AGENT_SESSION_HEADER"; const BACKEND_TIMEOUT_SECS_ENV: &str = "MESH_OPENAI_BACKEND_TIMEOUT_SECS"; +const MAX_AUDIO_MULTIPART_BODY_BYTES: usize = 64 * 1024 * 1024 + 1024 * 1024; /// Backend timeout override, in whole seconds. `0` disables the timeout. /// @@ -255,6 +268,17 @@ pub fn router_for_with_config( .route("/healthz", get(health)) .route("/readyz", get(ready)) .route("/v1/models", get(models)) + .route("/v1/embeddings", post(embeddings)) + .route("/v1/rerank", post(rerank)) + .route("/v1/audio/speech", post(audio_speech)) + .route( + "/v1/audio/transcriptions", + post(audio_transcriptions).layer(DefaultBodyLimit::max(MAX_AUDIO_MULTIPART_BODY_BYTES)), + ) + .route( + "/v1/audio/translations", + post(audio_translations).layer(DefaultBodyLimit::max(MAX_AUDIO_MULTIPART_BODY_BYTES)), + ) .route("/v1/chat/completions", post(chat_completions)) .route("/v1/completions", post(completions)) .route("/v1/responses", post(responses)) @@ -312,6 +336,241 @@ async fn models( })) } +/// Validate embedding input and preserve cancellation, usage, and lifecycle identity. +async fn embeddings( + State(state): State, + Extension(context): Extension, + payload: Result, JsonRejection>, +) -> Result { + let Json(request) = json_payload(payload)?; + request.validate()?; + let backend_context = OpenAiRequestContext::with_request_id(context.request_id); + let response: EmbeddingResponse = call_backend_with_context( + state.config.lifecycle_observer.clone(), + &context, + OpenAiBackendOperation::Embeddings, + "embeddings", + state.config.backend_timeout, + &backend_context, + state.backend.embeddings(request, backend_context.clone()), + ) + .await?; + state.response_completed( + &context, + OpenAiBackendOperation::Embeddings, + &response.usage, + ); + let usage = response.usage.clone(); + Ok(json_response_with_usage(response, &usage)) +} + +/// Validate query/documents before invoking the context-bound rerank backend. +async fn rerank( + State(state): State, + Extension(context): Extension, + payload: Result, JsonRejection>, +) -> Result { + let Json(request) = json_payload(payload)?; + request.validate()?; + let backend_context = OpenAiRequestContext::with_request_id(context.request_id); + let response: RerankResponse = call_backend_with_context( + state.config.lifecycle_observer.clone(), + &context, + OpenAiBackendOperation::Rerank, + "rerank", + state.config.backend_timeout, + &backend_context, + state.backend.rerank(request, backend_context.clone()), + ) + .await?; + state.response_completed(&context, OpenAiBackendOperation::Rerank, &response.usage); + let usage = response.usage.clone(); + Ok(json_response_with_usage(response, &usage)) +} + +/// Dispatch validated speech input and return binary audio without JSON wrapping. +async fn audio_speech( + State(state): State, + Extension(context): Extension, + payload: Result, JsonRejection>, +) -> Result { + let Json(request) = json_payload(payload)?; + request.validate()?; + let backend_context = OpenAiRequestContext::with_request_id(context.request_id); + let response = call_backend_with_context( + state.config.lifecycle_observer.clone(), + &context, + OpenAiBackendOperation::AudioSpeech, + "audio_speech", + state.config.backend_timeout, + &backend_context, + state.backend.audio_speech(request, backend_context.clone()), + ) + .await?; + audio_response(response) +} + +/// Validate the backend media type before placing audio bytes in the response. +fn audio_response(audio: AudioResponse) -> Result { + let content_type = HeaderValue::from_str(&audio.content_type) + .map_err(|_| OpenAiError::backend("audio backend returned an invalid content type"))?; + let mut response = Response::new(Body::from(audio.bytes)); + response + .headers_mut() + .insert(header::CONTENT_TYPE, content_type); + Ok(response) +} + +/// Handle source-language transcription through the shared multipart path. +async fn audio_transcriptions( + State(state): State, + Extension(context): Extension, + multipart: Result, +) -> Result { + audio_text_request(state, context, multipart_payload(multipart)?, false).await +} + +/// Select English translation without changing the multipart upload contract. +async fn audio_translations( + State(state): State, + Extension(context): Extension, + multipart: Result, +) -> Result { + audio_text_request(state, context, multipart_payload(multipart)?, true).await +} + +/// Convert extractor rejection into the frontend's structured invalid-request error. +fn multipart_payload(multipart: Result) -> OpenAiResult { + multipart.map_err(|error| { + OpenAiError::invalid_request(format!("invalid multipart request: {error}")) + }) +} + +/// Preserve payload-too-large status when a bounded multipart field cannot be read. +fn multipart_error(error: MultipartError, field: &str) -> OpenAiError { + if error.status() == StatusCode::PAYLOAD_TOO_LARGE { + OpenAiError::payload_too_large(format!("{field} is too large: {error}")) + } else { + OpenAiError::invalid_request(format!("invalid {field}: {error}")) + } +} + +/// Share upload validation and cancellation while retaining endpoint-specific dispatch. +async fn audio_text_request( + state: FrontendState, + context: OpenAiLifecycleContext, + multipart: Multipart, + translate: bool, +) -> Result { + let request = parse_audio_multipart(multipart).await?; + request.validate()?; + let response_format = request.response_format.clone(); + let backend_context = OpenAiRequestContext::with_request_id(context.request_id); + let operation = if translate { + OpenAiBackendOperation::AudioTranslation + } else { + OpenAiBackendOperation::AudioTranscription + }; + let response: AudioTranscriptionResponse = call_backend_with_context( + state.config.lifecycle_observer.clone(), + &context, + operation, + if translate { + "audio_translation" + } else { + "audio_transcription" + }, + state.config.backend_timeout, + &backend_context, + if translate { + state + .backend + .audio_translation(request, backend_context.clone()) + } else { + state + .backend + .audio_transcription(request, backend_context.clone()) + }, + ) + .await?; + if response_format == "text" { + Ok(( + [(header::CONTENT_TYPE, "text/plain; charset=utf-8")], + response.text, + ) + .into_response()) + } else { + Ok(Json(response).into_response()) + } +} + +/// Decode a bounded audio upload, rejecting duplicate recognized fields consistently. +async fn parse_audio_multipart( + mut multipart: Multipart, +) -> OpenAiResult { + let mut model = None; + let mut file = None; + let mut filename = None; + let mut language = None; + let mut prompt = None; + let mut response_format = None; + let mut temperature = None; + let mut seen_fields = std::collections::HashSet::new(); + + while let Some(field) = multipart + .next_field() + .await + .map_err(|error| multipart_error(error, "multipart body"))? + { + let name = field.name().unwrap_or_default().to_string(); + if matches!( + name.as_str(), + "model" | "file" | "language" | "prompt" | "response_format" | "temperature" + ) && !seen_fields.insert(name.clone()) + { + return Err(OpenAiError::invalid_request(format!( + "duplicate multipart {name} field" + ))); + } + if name == "file" { + filename = field.file_name().map(str::to_owned); + let bytes = field + .bytes() + .await + .map_err(|error| multipart_error(error, "audio file field"))?; + file = Some(bytes.to_vec()); + continue; + } + let value = field + .text() + .await + .map_err(|error| multipart_error(error, "multipart text field"))?; + match name.as_str() { + "model" => model = Some(value), + "language" => language = Some(value), + "prompt" => prompt = Some(value), + "response_format" => response_format = Some(value), + "temperature" => { + temperature = + Some(value.parse::().map_err(|_| { + OpenAiError::invalid_request("temperature must be a number") + })?); + } + _ => {} + } + } + + Ok(AudioTranscriptionRequest { + model: model.ok_or_else(|| OpenAiError::invalid_request("model field is required"))?, + file: file.ok_or_else(|| OpenAiError::invalid_request("file field is required"))?, + filename, + language, + prompt, + response_format: response_format.unwrap_or_else(|| "json".to_string()), + temperature, + }) +} + async fn chat_completions( State(state): State, Extension(context): Extension, @@ -968,12 +1227,18 @@ fn lifecycle_method(method: &Method) -> OpenAiRequestMethod { } } +/// Classify the endpoint path for lifecycle events without retaining query parameters. fn lifecycle_route(uri: &Uri) -> OpenAiFrontendRoute { match uri.path() { "/health" => OpenAiFrontendRoute::Health, "/healthz" => OpenAiFrontendRoute::Healthz, "/readyz" => OpenAiFrontendRoute::Readyz, "/v1/models" => OpenAiFrontendRoute::Models, + "/v1/embeddings" => OpenAiFrontendRoute::Embeddings, + "/v1/rerank" => OpenAiFrontendRoute::Rerank, + "/v1/audio/speech" => OpenAiFrontendRoute::AudioSpeech, + "/v1/audio/transcriptions" => OpenAiFrontendRoute::AudioTranscriptions, + "/v1/audio/translations" => OpenAiFrontendRoute::AudioTranslations, "/v1/chat/completions" => OpenAiFrontendRoute::ChatCompletions, "/v1/completions" => OpenAiFrontendRoute::Completions, "/v1/responses" => OpenAiFrontendRoute::Responses, diff --git a/crates/openai-frontend/src/router_tests.rs b/crates/openai-frontend/src/router_tests.rs index 07ef28388e..faf04f5e95 100644 --- a/crates/openai-frontend/src/router_tests.rs +++ b/crates/openai-frontend/src/router_tests.rs @@ -565,6 +565,85 @@ impl OpenAiBackend for FakeBackend { )), ]))) } + + /// Return indexed embedding fixtures with the requested encoding. + async fn embeddings( + &self, + request: EmbeddingsRequest, + _context: OpenAiRequestContext, + ) -> OpenAiResult { + let embeddings = (0..request.input.len()) + .map(|index| crate::Embedding { + values: vec![index as f32 + 1.0, -0.5], + index, + }) + .collect(); + Ok(EmbeddingResponse::from_embeddings( + request.model, + embeddings, + 7, + &request.encoding_format, + )) + } + + /// Return ranked fixtures honoring `top_n` and document inclusion. + async fn rerank( + &self, + request: RerankRequest, + _context: OpenAiRequestContext, + ) -> OpenAiResult { + let mut results = request + .documents + .iter() + .enumerate() + .map(|(index, document)| crate::RerankResult { + index, + relevance_score: (index + 1) as f32 / 10.0, + document: request.return_documents.then(|| document.clone()), + }) + .collect::>(); + results.sort_by(|left, right| right.relevance_score.total_cmp(&left.relevance_score)); + results.truncate(request.top_n.unwrap_or(results.len())); + Ok(RerankResponse { + id: "rerank_test".to_string(), + results, + usage: Usage::new(11, 0), + }) + } + + /// Return a binary speech fixture with the requested content type. + async fn audio_speech( + &self, + request: AudioSpeechRequest, + _context: OpenAiRequestContext, + ) -> OpenAiResult { + AudioResponse::new( + vec![0x52, 0x49, 0x46, 0x46], + request.response_format.content_type(), + ) + } + + /// Return a transcription fixture containing the upload length. + async fn audio_transcription( + &self, + request: AudioTranscriptionRequest, + _context: OpenAiRequestContext, + ) -> OpenAiResult { + Ok(AudioTranscriptionResponse { + text: format!("transcribed {} bytes", request.file.len()), + }) + } + + /// Return a translation fixture containing the upload length. + async fn audio_translation( + &self, + request: AudioTranscriptionRequest, + _context: OpenAiRequestContext, + ) -> OpenAiResult { + Ok(AudioTranscriptionResponse { + text: format!("translated {} bytes", request.file.len()), + }) + } } struct SlowBackend; @@ -775,6 +854,9 @@ async fn models_route_returns_model_list() { assert_eq!(body["data"][0]["id"], "org/repo:Q4_K_M"); } +#[path = "router_tests/non_chat.rs"] +mod non_chat; + #[tokio::test] async fn health_route_returns_liveness_probe() { let app = router_for(Arc::new(FakeBackend)); diff --git a/crates/openai-frontend/src/router_tests/non_chat.rs b/crates/openai-frontend/src/router_tests/non_chat.rs new file mode 100644 index 0000000000..0d87cb5f7a --- /dev/null +++ b/crates/openai-frontend/src/router_tests/non_chat.rs @@ -0,0 +1,298 @@ +use super::*; + +#[tokio::test] +/// Every recognized audio field has one authoritative value per upload. +async fn audio_upload_rejects_each_duplicate_field() { + let boundary = "duplicate-audio-field"; + for path in ["/v1/audio/transcriptions", "/v1/audio/translations"] { + for (field, value) in [ + ("model", "audio-model"), + ("file", "WAVE"), + ("language", "en"), + ("prompt", "words"), + ("response_format", "json"), + ("temperature", "0"), + ] { + let part = format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"{field}\"\r\n\r\n{value}\r\n" + ); + let mut body = part.repeat(2).into_bytes(); + body.extend_from_slice(&audio_multipart(boundary, "json")); + let response = post_audio_multipart(path, boundary, body).await; + assert_eq!( + response.status(), + StatusCode::BAD_REQUEST, + "{path}: {field}" + ); + let body = response_body_json(response).await; + assert_eq!( + body["error"]["message"], + format!("duplicate multipart {field} field") + ); + } + } +} + +#[tokio::test] +/// Transcription and translation share the same finite temperature contract. +async fn audio_upload_enforces_temperature_range_at_both_endpoints() { + let boundary = "audio-temperature"; + for path in ["/v1/audio/transcriptions", "/v1/audio/translations"] { + for (value, accepted) in [ + ("0", true), + ("1", true), + ("0.5", true), + ("1.1", false), + ("-0.1", false), + ("NaN", false), + ("inf", false), + ("-inf", false), + ] { + let mut body = format!("--{boundary}\r\nContent-Disposition: form-data; name=\"temperature\"\r\n\r\n{value}\r\n").into_bytes(); + body.extend_from_slice(&audio_multipart(boundary, "json")); + let response = post_audio_multipart(path, boundary, body).await; + assert_eq!( + response.status(), + if accepted { + StatusCode::OK + } else { + StatusCode::BAD_REQUEST + }, + "{path}: {value}" + ); + } + } +} + +#[tokio::test] +/// The HTTP envelope retains backend indexes, vector order and token accounting. +async fn embeddings_route_preserves_batch_order_and_usage() { + let response = post_json( + "/v1/embeddings", + json!({ + "model": "embed-model", + "input": ["first", "second"], + "encoding_format": "float" + }), + ) + .await; + + assert_eq!(response.status(), StatusCode::OK); + let body = response_body_json(response).await; + assert_eq!(body["object"], "list"); + assert_eq!(body["model"], "embed-model"); + assert_eq!(body["data"][0]["index"], 0); + assert_eq!(body["data"][0]["embedding"], json!([1.0, -0.5])); + assert_eq!(body["data"][1]["index"], 1); + assert_eq!(body["data"][1]["embedding"], json!([2.0, -0.5])); + assert_eq!(body["usage"]["prompt_tokens"], 7); + assert_eq!(body["usage"]["total_tokens"], 7); +} + +#[tokio::test] +/// Unsupported encodings return the standard API error instead of reaching inference. +async fn embeddings_route_returns_openai_error_for_invalid_format() { + let response = post_json( + "/v1/embeddings", + json!({ + "model": "embed-model", + "input": "hello", + "encoding_format": "hex" + }), + ) + .await; + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let body = response_body_json(response).await; + assert_eq!(body["error"]["type"], "invalid_request_error"); + assert_eq!(body["error"]["code"], "invalid_value"); +} + +#[tokio::test] +/// Sorting and top-n selection preserve each score's original document identity. +async fn rerank_route_sorts_limits_and_optionally_returns_documents() { + let response = post_json( + "/v1/rerank", + json!({ + "model": "rerank-model", + "query": "query", + "documents": ["one", {"text": "two", "title": "Two"}], + "top_n": 1, + "return_documents": true + }), + ) + .await; + + assert_eq!(response.status(), StatusCode::OK); + let body = response_body_json(response).await; + assert_eq!(body["id"], "rerank_test"); + assert_eq!(body["results"].as_array().unwrap().len(), 1); + assert_eq!(body["results"][0]["index"], 1); + assert_eq!(body["results"][0]["relevance_score"], 0.2); + assert_eq!(body["results"][0]["document"]["text"], "two"); + assert_eq!(body["usage"]["prompt_tokens"], 11); +} + +#[tokio::test] +/// Speech responses bypass JSON serialization and preserve the backend media type. +async fn audio_speech_route_returns_backend_bytes_and_content_type() { + let response = post_json( + "/v1/audio/speech", + json!({ + "model": "tts-model", + "input": "Hello", + "voice": "default", + "response_format": "wav" + }), + ) + .await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers()["content-type"], "audio/wav"); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(bytes.as_ref(), b"RIFF"); +} + +/// Build a deterministic upload body with a model, format and audio fixture. +fn audio_multipart(boundary: &str, response_format: &str) -> Vec { + format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\naudio-model\r\n\ + --{boundary}\r\nContent-Disposition: form-data; name=\"response_format\"\r\n\r\n{response_format}\r\n\ + --{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"sample.wav\"\r\n\ + Content-Type: audio/wav\r\n\r\nWAVE\r\n--{boundary}--\r\n" + ) + .into_bytes() +} + +/// Exercise the real router with a multipart body and matching content type. +async fn post_audio_multipart(path: &str, boundary: &str, body: Vec) -> Response { + router_for(Arc::new(FakeBackend)) + .oneshot( + Request::builder() + .method("POST") + .uri(path) + .header( + "content-type", + format!("multipart/form-data; boundary={boundary}"), + ) + .body(Body::from(body)) + .unwrap(), + ) + .await + .unwrap() +} + +#[tokio::test] +/// The requested transcription representation controls both bytes and content type. +async fn audio_transcription_supports_json_and_text_responses() { + let boundary = "mesh-audio-boundary"; + let json_response = post_audio_multipart( + "/v1/audio/transcriptions", + boundary, + audio_multipart(boundary, "json"), + ) + .await; + assert_eq!(json_response.status(), StatusCode::OK); + let body = response_body_json(json_response).await; + assert_eq!(body["text"], "transcribed 4 bytes"); + + let text_response = post_audio_multipart( + "/v1/audio/transcriptions", + boundary, + audio_multipart(boundary, "text"), + ) + .await; + assert_eq!(text_response.status(), StatusCode::OK); + assert_eq!( + text_response.headers()["content-type"], + "text/plain; charset=utf-8" + ); + assert_eq!( + response_body_text(text_response).await, + "transcribed 4 bytes" + ); +} + +#[tokio::test] +/// Translation must invoke its own backend operation rather than transcription. +async fn audio_translation_uses_translation_backend() { + let boundary = "mesh-audio-boundary"; + let response = post_audio_multipart( + "/v1/audio/translations", + boundary, + audio_multipart(boundary, "json"), + ) + .await; + + assert_eq!(response.status(), StatusCode::OK); + let body = response_body_json(response).await; + assert_eq!(body["text"], "translated 4 bytes"); +} + +#[tokio::test] +/// Binary upload limits remain independent of JSON endpoint limits. +async fn audio_upload_uses_its_dedicated_body_limit() { + let boundary = "mesh-large-audio-boundary"; + let file_bytes = vec![0x2a; OpenAiFrontendConfig::default().max_request_body_bytes + 1]; + let mut body = format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\naudio-model\r\n\ + --{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"sample.wav\"\r\n\ + Content-Type: audio/wav\r\n\r\n" + ) + .into_bytes(); + body.extend_from_slice(&file_bytes); + body.extend_from_slice(format!("\r\n--{boundary}--\r\n").as_bytes()); + + let response = post_audio_multipart("/v1/audio/transcriptions", boundary, body).await; + + assert_eq!(response.status(), StatusCode::OK); + let response = response_body_json(response).await; + assert_eq!( + response["text"], + format!("transcribed {} bytes", file_bytes.len()) + ); +} + +#[tokio::test] +/// Malformed uploads must produce typed API errors, not framework-specific bodies. +async fn malformed_audio_multipart_uses_openai_error_envelope() { + let response = router_for(Arc::new(FakeBackend)) + .oneshot( + Request::builder() + .method("POST") + .uri("/v1/audio/transcriptions") + .header("content-type", "multipart/form-data") + .body(Body::from("not multipart")) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let body = response_body_json(response).await; + assert_eq!(body["error"]["type"], "invalid_request_error"); +} + +#[tokio::test] +/// Ambiguous model selection is rejected before the backend receives an upload. +async fn duplicate_audio_model_field_is_rejected() { + let boundary = "mesh-audio-boundary"; + let body = format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\naudio-model\r\n\ + --{boundary}\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\nother-model\r\n\ + --{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"sample.wav\"\r\n\ + Content-Type: audio/wav\r\n\r\nWAVE\r\n--{boundary}--\r\n" + ); + for path in ["/v1/audio/transcriptions", "/v1/audio/translations"] { + let response = post_audio_multipart(path, boundary, body.as_bytes().to_vec()).await; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let payload = response_body_json(response).await; + assert_eq!(payload["error"]["type"], "invalid_request_error"); + assert!( + payload["error"]["message"] + .as_str() + .unwrap() + .contains("duplicate multipart model field") + ); + } +} diff --git a/crates/skippy-ffi/build.rs b/crates/skippy-ffi/build.rs index d9e66567fc..88452e7052 100644 --- a/crates/skippy-ffi/build.rs +++ b/crates/skippy-ffi/build.rs @@ -167,24 +167,33 @@ fn main() { println!("cargo:rustc-link-lib=static=llama-common-base"); println!("cargo:rustc-link-lib=static=llama"); println!("cargo:rustc-link-lib=static=ggml"); - let has_cuda = static_archive_exists( + let has_cuda = configured_backend_archive( &build_dir, + &cmake_cache, + backend == "cuda", + "GGML_CUDA", "ggml/src/ggml-cuda/libggml-cuda.a", "ggml/src/ggml-cuda/ggml-cuda.lib", ); if has_cuda { println!("cargo:rustc-link-lib=static=ggml-cuda"); } - let has_hip = static_archive_exists( + let has_hip = configured_backend_archive( &build_dir, + &cmake_cache, + backend == "rocm" || backend == "hip", + "GGML_HIP", "ggml/src/ggml-hip/libggml-hip.a", "ggml/src/ggml-hip/ggml-hip.lib", ); if has_hip { println!("cargo:rustc-link-lib=static=ggml-hip"); } - let has_vulkan = static_archive_exists( + let has_vulkan = configured_backend_archive( &build_dir, + &cmake_cache, + backend == "vulkan", + "GGML_VULKAN", "ggml/src/ggml-vulkan/libggml-vulkan.a", "ggml/src/ggml-vulkan/ggml-vulkan.lib", ); @@ -192,18 +201,24 @@ fn main() { println!("cargo:rustc-link-lib=static=ggml-vulkan"); } println!("cargo:rustc-link-lib=static=ggml-cpu"); - if static_archive_exists( - &build_dir, - "ggml/src/ggml-blas/libggml-blas.a", - "ggml/src/ggml-blas/ggml-blas.lib", - ) { + if cmake_bool_enabled(&cmake_cache, "GGML_BLAS") + && static_archive_exists( + &build_dir, + "ggml/src/ggml-blas/libggml-blas.a", + "ggml/src/ggml-blas/ggml-blas.lib", + ) + { println!("cargo:rustc-link-lib=static=ggml-blas"); } - if static_archive_exists( + let has_metal = configured_backend_archive( &build_dir, + &cmake_cache, + backend == "metal", + "GGML_METAL", "ggml/src/ggml-metal/libggml-metal.a", "ggml/src/ggml-metal/ggml-metal.lib", - ) { + ); + if has_metal { println!("cargo:rustc-link-lib=static=ggml-metal"); } println!("cargo:rustc-link-lib=static=ggml-base"); @@ -212,11 +227,7 @@ fn main() { link_apple_openmp_libs(&cmake_cache); println!("cargo:rustc-link-lib=c++"); println!("cargo:rustc-link-lib=framework=Accelerate"); - if static_archive_exists( - &build_dir, - "ggml/src/ggml-metal/libggml-metal.a", - "ggml/src/ggml-metal/ggml-metal.lib", - ) { + if has_metal { println!("cargo:rustc-link-lib=framework=Foundation"); println!("cargo:rustc-link-lib=framework=Metal"); println!("cargo:rustc-link-lib=framework=MetalKit"); @@ -434,6 +445,45 @@ fn static_archive_exists( build_dir.join(unix_archive).exists() || build_dir.join(msvc_archive).exists() } +/// Read CMake booleans while tolerating platform line endings and whitespace. +fn cmake_bool_enabled(cache: &std::path::Path, key: &str) -> bool { + let Ok(contents) = std::fs::read_to_string(cache) else { + return false; + }; + cmake_cache_bool(&contents, key) +} + +/// Resolve an enabled backend archive without trusting stale native build outputs. +fn configured_backend_archive( + build_dir: &std::path::Path, + cmake_cache: &std::path::Path, + selected_backend: bool, + cmake_key: &str, + unix_archive: &str, + msvc_archive: &str, +) -> bool { + let configured = cmake_bool_enabled(cmake_cache, cmake_key); + if !selected_backend { + assert!( + !configured, + "staged backend mismatch: {cmake_key}=ON in {} but LLAMA_STAGE_BACKEND does not select it", + cmake_cache.display() + ); + return false; + } + assert!( + configured, + "selected backend requires {cmake_key}=ON in {}", + cmake_cache.display() + ); + assert!( + static_archive_exists(build_dir, unix_archive, msvc_archive), + "selected backend archive is missing from {}", + build_dir.display() + ); + true +} + fn link_linux_cuda_libs(cmake_cache: &std::path::Path) { for (cache_key, lib) in [ ("CUDA_cuda_driver_LIBRARY", "cuda"), @@ -686,8 +736,9 @@ fn cmake_cache_value(cache: &str, key: &str) -> Option { }) } +/// Interpret affirmative CMake cache values after trimming platform whitespace. fn cmake_cache_bool(cache: &str, key: &str) -> bool { cmake_cache_value(cache, key) - .map(|value| matches!(value.as_str(), "ON" | "TRUE" | "1")) + .map(|value| matches!(value.trim(), "ON" | "TRUE" | "1")) .unwrap_or(false) } diff --git a/crates/skippy-ffi/src/abi.rs b/crates/skippy-ffi/src/abi.rs index df5e0e9b37..ed04ed2928 100644 --- a/crates/skippy-ffi/src/abi.rs +++ b/crates/skippy-ffi/src/abi.rs @@ -19,7 +19,65 @@ pub const FEATURE_KV_EVENTS: u64 = 1 << 33; pub const FEATURE_DEVICE_EVENTS: u64 = 1 << 34; pub const FEATURE_DIAGNOSTIC_EVENTS: u64 = 1 << 35; pub const FEATURE_UNLOAD_EVENTS: u64 = 1 << 36; +/// Full-model workloads use a separate bit from all runtime-event families. +pub const FEATURE_NON_CHAT_WORKLOADS: u64 = 1 << 37; pub const MODEL_TENSOR_SOURCE_V1_ABI_VERSION: u32 = 1; +pub const WORKLOAD_INFO_V1_ABI_VERSION: u32 = 1; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +#[repr(i32)] +pub enum WorkloadKind { + #[default] + CausalGeneration = 0, + Embedding = 1, + Rerank = 2, + EncoderDecoder = 3, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +#[repr(i32)] +pub enum WorkloadPooling { + Unspecified = -1, + #[default] + None = 0, + Mean = 1, + Cls = 2, + Last = 3, + Rank = 4, +} + +#[repr(C)] +#[derive(Debug, Clone, Copy)] +pub struct WorkloadInfoV1 { + pub abi_version: u32, + pub struct_size: u32, + pub kind: WorkloadKind, + pub pooling: WorkloadPooling, + pub output_dimensions: u32, + pub classifier_outputs: u32, + pub has_encoder: bool, + pub has_decoder: bool, + pub full_model_only: bool, + pub reserved0: u8, +} + +impl Default for WorkloadInfoV1 { + /// Initialize the workload descriptor with its ABI size and version. + fn default() -> Self { + Self { + abi_version: WORKLOAD_INFO_V1_ABI_VERSION, + struct_size: std::mem::size_of::() as u32, + kind: WorkloadKind::default(), + pooling: WorkloadPooling::default(), + output_dimensions: 0, + classifier_outputs: 0, + has_encoder: false, + has_decoder: false, + full_model_only: true, + reserved0: 0, + } + } +} pub type ModelReadTensorF32Callback = Option< unsafe extern "C" fn( diff --git a/crates/skippy-ffi/src/dynamic.rs b/crates/skippy-ffi/src/dynamic.rs index d2b8492596..d6538fd193 100644 --- a/crates/skippy-ffi/src/dynamic.rs +++ b/crates/skippy-ffi/src/dynamic.rs @@ -8,13 +8,14 @@ use crate::{ ABI_VERSION_MAJOR, ABI_VERSION_MINOR, ABI_VERSION_PATCH, AbiVersion, ActivationBoundaryDesc, ActivationDesc, BackendDevice, Error, GenerationSignalWindow, IterationRequest, KvPageDesc, LlamaLogCallback, LlamaModelQuantizeParams, Model, ModelInfo, ModelTensorSourceV1, MtmdBitmap, - MtmdContext, MtmdContextParams, MtmdDecoderPos, MtmdHelperBitmapWrapper, MtmdHelperInitOpt, - MtmdHelperVideo, MtmdInputChunkType, MtmdInputChunks, MtmdInputText, NativeMtpDraft, - NativeRuntimeLoadError, NgramCache, Opaque, RuntimeConfig, SamplingConfig, Session, - SkippyDecodeStepSampledMtpFn, SkippyModelAttachMtpDraftModelFn, SkippyRuntimeEventReporterV1, - SlicePlan, StagePlan, StagePlanDescV1, StagePlanProfileDescV1, StagePlanStateDescV1, - StagePlanStringRefV1, StagePlanValueDescV1, StagePlanValueKind, StagePlanner, - StagePlannerConfigV1, Status, TensorInfo, TokenSignal, runtime_abi_supported, + MtmdContext, MtmdContextParams, MtmdDecoderPos, MtmdGenAudioInfo, MtmdHelperBitmapWrapper, + MtmdHelperGenAudio, MtmdHelperGenAudioInput, MtmdHelperInitOpt, MtmdHelperVideo, + MtmdInputChunkType, MtmdInputChunks, MtmdInputText, NativeMtpDraft, NativeRuntimeLoadError, + NgramCache, Opaque, RuntimeConfig, SamplingConfig, Session, SkippyDecodeStepSampledMtpFn, + SkippyModelAttachMtpDraftModelFn, SkippyRuntimeEventReporterV1, SlicePlan, StagePlan, + StagePlanDescV1, StagePlanProfileDescV1, StagePlanStateDescV1, StagePlanStringRefV1, + StagePlanValueDescV1, StagePlanValueKind, StagePlanner, StagePlannerConfigV1, Status, + TensorInfo, TokenSignal, WorkloadInfoV1, runtime_abi_supported, }; static SYMBOLS: OnceLock = OnceLock::new(); @@ -164,6 +165,8 @@ macro_rules! dynamic_symbols { } dynamic_symbols! { + llama_get_embeddings_ith(ctx: *mut Opaque, index: i32) -> *mut f32; + llama_set_embeddings(ctx: *mut Opaque, embeddings: bool); llama_log_set(log_callback: LlamaLogCallback, user_data: *mut c_void); ggml_log_set(log_callback: LlamaLogCallback, user_data: *mut c_void); llama_model_quantize_default_params() -> LlamaModelQuantizeParams; @@ -183,6 +186,7 @@ dynamic_symbols! { skippy_model_llama_model(model: *const Model) -> *const Opaque; skippy_model_output_activation_boundary(model: *const Model, out_desc: *mut ActivationBoundaryDesc) -> bool; skippy_model_input_activation_boundary(model: *const Model, out_desc: *mut ActivationBoundaryDesc) -> bool; + skippy_model_workload_info_v1(model: *const Model, out_info: *mut WorkloadInfoV1, out_error: *mut *mut Error) -> Status; skippy_session_create(model: *mut Model, out_session: *mut *mut Session, out_error: *mut *mut Error) -> Status; skippy_session_create_from_resident_prefix(model: *mut Model, cache_seq_id: i32, token_ids: *const i32, token_count: usize, out_session: *mut *mut Session, out_error: *mut *mut Error) -> Status; skippy_session_llama_context(session: *mut Session) -> *mut Opaque; @@ -194,6 +198,9 @@ dynamic_symbols! { skippy_session_set_position(session: *mut Session, n_past: i32, out_error: *mut *mut Error) -> Status; skippy_session_sample_current(session: *mut Session, sampling: *const SamplingConfig, out_predicted_token: *mut i32, out_error: *mut *mut Error) -> Status; skippy_session_configure_chat_sampling(session: *mut Session, sampling: *const SamplingConfig, metadata_json: *const c_char, prompt_token_count: u64, out_error: *mut *mut Error) -> Status; + skippy_session_embed(session: *mut Session, token_ids: *const i32, token_count: usize, output: *mut f32, output_capacity: usize, out_dimensions: *mut usize, out_error: *mut *mut Error) -> Status; + skippy_session_rerank(session: *mut Session, query: *const c_char, document: *const c_char, out_score: *mut f32, out_token_count: *mut usize, out_error: *mut *mut Error) -> Status; + skippy_session_encode_prompt(session: *mut Session, token_ids: *const i32, token_count: usize, out_decoder_start_token: *mut i32, out_error: *mut *mut Error) -> Status; skippy_session_reset(session: *mut Session, out_error: *mut *mut Error) -> Status; skippy_session_free(session: *mut Session, out_error: *mut *mut Error) -> Status; skippy_prefill_chunk(session: *mut Session, token_ids: *const i32, token_count: usize, input_activations: *const c_void, input_activation_bytes: usize, output_activations: *mut c_void, output_activation_capacity: usize, out_output_activation_bytes: *mut usize, out_error: *mut *mut Error) -> Status; @@ -253,6 +260,14 @@ dynamic_symbols! { mtmd_default_marker() -> *const c_char; mtmd_helper_log_set(log_callback: LlamaLogCallback, user_data: *mut c_void); mtmd_context_params_default() -> MtmdContextParams; + mtmd_gen_audio_get_info(ctx: *const MtmdContext) -> MtmdGenAudioInfo; + mtmd_helper_gen_audio_init(lctx: *mut Opaque, mctx: *mut MtmdContext) -> *mut MtmdHelperGenAudio; + mtmd_helper_gen_audio_free(ctx: *mut MtmdHelperGenAudio); + mtmd_helper_gen_audio_reset(ctx: *mut MtmdHelperGenAudio); + mtmd_helper_gen_audio_set_input(ctx: *mut MtmdHelperGenAudio, input: *const MtmdHelperGenAudioInput) -> i32; + mtmd_helper_gen_audio_step_prompt(ctx: *mut MtmdHelperGenAudio, n_batch: i32) -> i32; + mtmd_helper_gen_audio_step_gen(ctx: *mut MtmdHelperGenAudio, sampled: i32, h_state_in: *const f32, h_state_out: *mut *const f32, out_stop: *mut bool) -> i32; + mtmd_helper_gen_audio_get_output(ctx: *mut MtmdHelperGenAudio, out_sample_rate: *mut i32, out_data: *mut *const c_char, out_data_len: *mut usize, out_n_samples: *mut i64) -> i32; mtmd_init_from_file(mmproj_fname: *const c_char, text_model: *const Opaque, ctx_params: MtmdContextParams) -> *mut MtmdContext; mtmd_free(ctx: *mut MtmdContext); mtmd_helper_init_opt_default() -> MtmdHelperInitOpt; diff --git a/crates/skippy-ffi/src/lib.rs b/crates/skippy-ffi/src/lib.rs index 58ee6bf20c..b92137c3be 100644 --- a/crates/skippy-ffi/src/lib.rs +++ b/crates/skippy-ffi/src/lib.rs @@ -5,7 +5,7 @@ mod dynamic_library; // without compiling the crate to determine native-runtime compatibility. pub const ABI_VERSION_MAJOR: u32 = 0; pub const ABI_VERSION_MINOR: u32 = 1; -pub const ABI_VERSION_PATCH: u32 = 54; +pub const ABI_VERSION_PATCH: u32 = 55; mod abi; mod activation; @@ -30,15 +30,16 @@ pub use abi::{ FEATURE_ACTIVATION_BOUNDARY, FEATURE_BACKEND_DEVICES, FEATURE_DEVICE_EVENTS, FEATURE_DIAGNOSTIC_EVENTS, FEATURE_INKLING_MTP_MM, FEATURE_ITERATION_BATCH, FEATURE_KV_EVENTS, FEATURE_MODEL_LOAD_EVENTS_V2, FEATURE_MODEL_SOURCE, FEATURE_NATIVE_MTP_N1, - FEATURE_NGRAM_CACHE_DRAFT, FEATURE_RUNTIME_EVENT_REPORTER, FEATURE_RUNTIME_EVENTS, - FEATURE_STAGE_PLAN, FEATURE_UNLOAD_EVENTS, IterationRequest, LlamaLogCallback, LoadMode, - MODEL_TENSOR_SOURCE_V1_ABI_VERSION, Model, ModelImatrixEntryV1, ModelInfo, - ModelReadTensorF32Callback, ModelTensorSourceV1, MtmdProgressCallback, MtpSource, NgramCache, - Opaque, RuntimeConfig, Session, SkippyDecodeStepSampledMtpFn, SkippyModelAttachMtpDraftModelFn, - SkippyRuntimeEventCallback, SkippyRuntimeEventCategory, SkippyRuntimeEventEmitterKind, - SkippyRuntimeEventFailureCode, SkippyRuntimeEventKind, SkippyRuntimeEventProgressUnit, - SkippyRuntimeEventReporterV1, SkippyRuntimeEventV1, SlicePlan, Status, TRISTATE_AUTO, - TRISTATE_FALSE, TRISTATE_TRUE, TensorRole, runtime_abi_supported, + FEATURE_NGRAM_CACHE_DRAFT, FEATURE_NON_CHAT_WORKLOADS, FEATURE_RUNTIME_EVENT_REPORTER, + FEATURE_RUNTIME_EVENTS, FEATURE_STAGE_PLAN, FEATURE_UNLOAD_EVENTS, IterationRequest, + LlamaLogCallback, LoadMode, MODEL_TENSOR_SOURCE_V1_ABI_VERSION, Model, ModelImatrixEntryV1, + ModelInfo, ModelReadTensorF32Callback, ModelTensorSourceV1, MtmdProgressCallback, MtpSource, + NgramCache, Opaque, RuntimeConfig, Session, SkippyDecodeStepSampledMtpFn, + SkippyModelAttachMtpDraftModelFn, SkippyRuntimeEventCallback, SkippyRuntimeEventCategory, + SkippyRuntimeEventEmitterKind, SkippyRuntimeEventFailureCode, SkippyRuntimeEventKind, + SkippyRuntimeEventProgressUnit, SkippyRuntimeEventReporterV1, SkippyRuntimeEventV1, SlicePlan, + Status, TRISTATE_AUTO, TRISTATE_FALSE, TRISTATE_TRUE, TensorRole, WORKLOAD_INFO_V1_ABI_VERSION, + WorkloadInfoV1, WorkloadKind, WorkloadPooling, runtime_abi_supported, }; pub use activation::{ ACTIVATION_FLAG_GEMMA3N_ALTUP, ACTIVATION_FLAG_GLM_DSA_TOP_K, ACTIVATION_FLAG_INKLING_MTP_EMBD, @@ -50,9 +51,10 @@ pub use model::{ LlamaModelKvOverrideValue, LlamaModelQuantizeParams, LlamaModelTensorOverride, }; pub use multimodal::{ - MtmdBitmap, MtmdContext, MtmdContextParams, MtmdDecoderPos, MtmdHelperBitmapWrapper, - MtmdHelperInitOpt, MtmdHelperVideo, MtmdHelperVideoInitParams, MtmdInputChunkType, - MtmdInputChunks, MtmdInputText, + MtmdBitmap, MtmdContext, MtmdContextParams, MtmdDecoderPos, MtmdGenAudioInfo, MtmdGenAudioType, + MtmdHelperBitmapWrapper, MtmdHelperGenAudio, MtmdHelperGenAudioInput, + MtmdHelperGenAudioOutputType, MtmdHelperInitOpt, MtmdHelperVideo, MtmdHelperVideoInitParams, + MtmdInputChunkType, MtmdInputChunks, MtmdInputText, }; pub use runtime::{ NativeRuntimeLoadError, abi_features, llama_model_is_diffusion, llama_model_is_hybrid, @@ -87,21 +89,24 @@ pub use runtime::skippy_abi_features; #[cfg(feature = "dynamic-runtime")] pub use dynamic::{ - ggml_log_set, llama_log_set, llama_model_quantize, llama_model_quantize_default_params, - load_native_runtime_libraries, load_native_runtime_library, mtmd_bitmap_free, - mtmd_context_params_default, mtmd_decode_use_mrope, mtmd_default_marker, mtmd_free, + ggml_log_set, llama_get_embeddings_ith, llama_log_set, llama_model_quantize, + llama_model_quantize_default_params, llama_set_embeddings, load_native_runtime_libraries, + load_native_runtime_library, mtmd_bitmap_free, mtmd_context_params_default, + mtmd_decode_use_mrope, mtmd_default_marker, mtmd_free, mtmd_gen_audio_get_info, mtmd_helper_bitmap_init_from_buf, mtmd_helper_eval_chunk_single, mtmd_helper_eval_chunks, - mtmd_helper_get_n_pos, mtmd_helper_get_n_tokens, mtmd_helper_image_get_decoder_pos, - mtmd_helper_init_opt_default, mtmd_helper_log_set, mtmd_helper_video_free, mtmd_init_from_file, - mtmd_input_chunk_get_n_tokens, mtmd_input_chunk_get_tokens_image, - mtmd_input_chunk_get_tokens_text, mtmd_input_chunk_get_type, mtmd_input_chunks_free, - mtmd_input_chunks_get, mtmd_input_chunks_init, mtmd_input_chunks_size, mtmd_tokenize, - native_runtime_loaded, skippy_abi_features_optional, skippy_apply_chat_template_json, - skippy_backend_device_at, skippy_backend_device_count, skippy_decode_batch_sampled, - skippy_decode_step_frame_batch_sampled, skippy_decode_step_frame_sampled, - skippy_decode_step_frame_sampled_mtp, skippy_decode_step_sampled, - skippy_decode_step_sampled_mtp, skippy_decode_step_sampled_mtp_fn, skippy_detokenize, - skippy_error_free, skippy_export_full_state, skippy_export_kv_page, + mtmd_helper_gen_audio_free, mtmd_helper_gen_audio_get_output, mtmd_helper_gen_audio_init, + mtmd_helper_gen_audio_reset, mtmd_helper_gen_audio_set_input, mtmd_helper_gen_audio_step_gen, + mtmd_helper_gen_audio_step_prompt, mtmd_helper_get_n_pos, mtmd_helper_get_n_tokens, + mtmd_helper_image_get_decoder_pos, mtmd_helper_init_opt_default, mtmd_helper_log_set, + mtmd_helper_video_free, mtmd_init_from_file, mtmd_input_chunk_get_n_tokens, + mtmd_input_chunk_get_tokens_image, mtmd_input_chunk_get_tokens_text, mtmd_input_chunk_get_type, + mtmd_input_chunks_free, mtmd_input_chunks_get, mtmd_input_chunks_init, mtmd_input_chunks_size, + mtmd_tokenize, native_runtime_loaded, skippy_abi_features_optional, + skippy_apply_chat_template_json, skippy_backend_device_at, skippy_backend_device_count, + skippy_decode_batch_sampled, skippy_decode_step_frame_batch_sampled, + skippy_decode_step_frame_sampled, skippy_decode_step_frame_sampled_mtp, + skippy_decode_step_sampled, skippy_decode_step_sampled_mtp, skippy_decode_step_sampled_mtp_fn, + skippy_detokenize, skippy_error_free, skippy_export_full_state, skippy_export_kv_page, skippy_export_recurrent_state, skippy_export_state, skippy_import_full_state, skippy_import_kv_page, skippy_import_recurrent_state, skippy_import_state, skippy_iteration_batch_sampled, skippy_model_attach_mtp_draft_model_fn, skippy_model_free, @@ -110,26 +115,27 @@ pub use dynamic::{ skippy_model_llama_model, skippy_model_open, skippy_model_open_from_parts, skippy_model_open_from_parts_with_events_fn, skippy_model_open_from_source, skippy_model_open_with_events_fn, skippy_model_output_activation_boundary, - skippy_ngram_cache_append, skippy_ngram_cache_create, skippy_ngram_cache_draft, - skippy_ngram_cache_free, skippy_ngram_cache_reset, skippy_parse_chat_response_json, - skippy_prefill_chunk, skippy_prefill_chunk_frame, skippy_prefill_chunk_frame_sampled, - skippy_prefill_chunk_frame_sampled_with_positions, skippy_prefill_chunk_frame_with_positions, - skippy_retire_verify_checkpoint, skippy_session_batch_size, - skippy_session_begin_external_decode, skippy_session_configure_chat_sampling, - skippy_session_copy_output_activation_frame, skippy_session_create, - skippy_session_create_from_resident_prefix, skippy_session_drop_sequence, + skippy_model_workload_info_v1, skippy_ngram_cache_append, skippy_ngram_cache_create, + skippy_ngram_cache_draft, skippy_ngram_cache_free, skippy_ngram_cache_reset, + skippy_parse_chat_response_json, skippy_prefill_chunk, skippy_prefill_chunk_frame, + skippy_prefill_chunk_frame_sampled, skippy_prefill_chunk_frame_sampled_with_positions, + skippy_prefill_chunk_frame_with_positions, skippy_retire_verify_checkpoint, + skippy_session_batch_size, skippy_session_begin_external_decode, + skippy_session_configure_chat_sampling, skippy_session_copy_output_activation_frame, + skippy_session_create, skippy_session_create_from_resident_prefix, + skippy_session_drop_sequence, skippy_session_embed, skippy_session_encode_prompt, skippy_session_end_external_decode, skippy_session_free, skippy_session_last_token_signal, skippy_session_llama_context, skippy_session_memory_used_cells, skippy_session_position, - skippy_session_reset, skippy_session_restore_prefix, skippy_session_sample_current, - skippy_session_save_prefix, skippy_session_sequence_id, skippy_session_set_position, - skippy_session_signal_window, skippy_slice_plan_add_layer_range, skippy_slice_plan_create, - skippy_slice_plan_free, skippy_stage_plan_describe_v1, skippy_stage_plan_free, - skippy_stage_plan_profile_at_v1, skippy_stage_plan_resident_tensor_at_v1, - skippy_stage_plan_state_at_v1, skippy_stage_plan_string_v1, - skippy_stage_plan_validate_chain_v1, skippy_stage_plan_value_at_v1, - skippy_stage_planner_create_v1, skippy_stage_planner_free, skippy_stage_planner_realize_v1, - skippy_token_is_eog, skippy_tokenize, skippy_trim_session, skippy_verify_tokens, - skippy_verify_tokens_frame_sampled, skippy_write_gguf_from_parts, + skippy_session_rerank, skippy_session_reset, skippy_session_restore_prefix, + skippy_session_sample_current, skippy_session_save_prefix, skippy_session_sequence_id, + skippy_session_set_position, skippy_session_signal_window, skippy_slice_plan_add_layer_range, + skippy_slice_plan_create, skippy_slice_plan_free, skippy_stage_plan_describe_v1, + skippy_stage_plan_free, skippy_stage_plan_profile_at_v1, + skippy_stage_plan_resident_tensor_at_v1, skippy_stage_plan_state_at_v1, + skippy_stage_plan_string_v1, skippy_stage_plan_validate_chain_v1, + skippy_stage_plan_value_at_v1, skippy_stage_planner_create_v1, skippy_stage_planner_free, + skippy_stage_planner_realize_v1, skippy_token_is_eog, skippy_tokenize, skippy_trim_session, + skippy_verify_tokens, skippy_verify_tokens_frame_sampled, skippy_write_gguf_from_parts, skippy_write_gguf_metadata_from_parts, skippy_write_slice_gguf, }; @@ -140,15 +146,19 @@ pub use dynamic::{ #[cfg(not(feature = "dynamic-runtime"))] pub use static_bindings::{ - ggml_log_set, llama_log_set, llama_model_quantize, llama_model_quantize_default_params, - mtmd_bitmap_free, mtmd_context_params_default, mtmd_decode_use_mrope, mtmd_default_marker, - mtmd_free, mtmd_helper_bitmap_init_from_buf, mtmd_helper_eval_chunk_single, - mtmd_helper_eval_chunks, mtmd_helper_get_n_pos, mtmd_helper_get_n_tokens, - mtmd_helper_image_get_decoder_pos, mtmd_helper_init_opt_default, mtmd_helper_log_set, - mtmd_helper_video_free, mtmd_init_from_file, mtmd_input_chunk_get_n_tokens, - mtmd_input_chunk_get_tokens_image, mtmd_input_chunk_get_tokens_text, mtmd_input_chunk_get_type, - mtmd_input_chunks_free, mtmd_input_chunks_get, mtmd_input_chunks_init, mtmd_input_chunks_size, - mtmd_tokenize, skippy_abi_features, skippy_apply_chat_template_json, skippy_backend_device_at, + ggml_log_set, llama_get_embeddings_ith, llama_log_set, llama_model_quantize, + llama_model_quantize_default_params, llama_set_embeddings, mtmd_bitmap_free, + mtmd_context_params_default, mtmd_decode_use_mrope, mtmd_default_marker, mtmd_free, + mtmd_gen_audio_get_info, mtmd_helper_bitmap_init_from_buf, mtmd_helper_eval_chunk_single, + mtmd_helper_eval_chunks, mtmd_helper_gen_audio_free, mtmd_helper_gen_audio_get_output, + mtmd_helper_gen_audio_init, mtmd_helper_gen_audio_reset, mtmd_helper_gen_audio_set_input, + mtmd_helper_gen_audio_step_gen, mtmd_helper_gen_audio_step_prompt, mtmd_helper_get_n_pos, + mtmd_helper_get_n_tokens, mtmd_helper_image_get_decoder_pos, mtmd_helper_init_opt_default, + mtmd_helper_log_set, mtmd_helper_video_free, mtmd_init_from_file, + mtmd_input_chunk_get_n_tokens, mtmd_input_chunk_get_tokens_image, + mtmd_input_chunk_get_tokens_text, mtmd_input_chunk_get_type, mtmd_input_chunks_free, + mtmd_input_chunks_get, mtmd_input_chunks_init, mtmd_input_chunks_size, mtmd_tokenize, + skippy_abi_features, skippy_apply_chat_template_json, skippy_backend_device_at, skippy_backend_device_count, skippy_decode_batch_sampled, skippy_decode_step_frame_batch_sampled, skippy_decode_step_frame_sampled, skippy_decode_step_frame_sampled_mtp, skippy_decode_step_sampled, @@ -160,25 +170,26 @@ pub use static_bindings::{ skippy_model_info_tensor_count, skippy_model_input_activation_boundary, skippy_model_llama_model, skippy_model_open, skippy_model_open_from_parts, skippy_model_open_from_source, skippy_model_output_activation_boundary, - skippy_ngram_cache_append, skippy_ngram_cache_create, skippy_ngram_cache_draft, - skippy_ngram_cache_free, skippy_ngram_cache_reset, skippy_parse_chat_response_json, - skippy_prefill_chunk, skippy_prefill_chunk_frame, skippy_prefill_chunk_frame_sampled, - skippy_prefill_chunk_frame_sampled_with_positions, skippy_prefill_chunk_frame_with_positions, - skippy_retire_verify_checkpoint, skippy_session_batch_size, - skippy_session_begin_external_decode, skippy_session_configure_chat_sampling, - skippy_session_copy_output_activation_frame, skippy_session_create, - skippy_session_create_from_resident_prefix, skippy_session_drop_sequence, + skippy_model_workload_info_v1, skippy_ngram_cache_append, skippy_ngram_cache_create, + skippy_ngram_cache_draft, skippy_ngram_cache_free, skippy_ngram_cache_reset, + skippy_parse_chat_response_json, skippy_prefill_chunk, skippy_prefill_chunk_frame, + skippy_prefill_chunk_frame_sampled, skippy_prefill_chunk_frame_sampled_with_positions, + skippy_prefill_chunk_frame_with_positions, skippy_retire_verify_checkpoint, + skippy_session_batch_size, skippy_session_begin_external_decode, + skippy_session_configure_chat_sampling, skippy_session_copy_output_activation_frame, + skippy_session_create, skippy_session_create_from_resident_prefix, + skippy_session_drop_sequence, skippy_session_embed, skippy_session_encode_prompt, skippy_session_end_external_decode, skippy_session_free, skippy_session_last_token_signal, skippy_session_llama_context, skippy_session_memory_used_cells, skippy_session_position, - skippy_session_reset, skippy_session_restore_prefix, skippy_session_sample_current, - skippy_session_save_prefix, skippy_session_sequence_id, skippy_session_set_position, - skippy_session_signal_window, skippy_slice_plan_add_layer_range, skippy_slice_plan_create, - skippy_slice_plan_free, skippy_stage_plan_describe_v1, skippy_stage_plan_free, - skippy_stage_plan_profile_at_v1, skippy_stage_plan_resident_tensor_at_v1, - skippy_stage_plan_state_at_v1, skippy_stage_plan_string_v1, - skippy_stage_plan_validate_chain_v1, skippy_stage_plan_value_at_v1, - skippy_stage_planner_create_v1, skippy_stage_planner_free, skippy_stage_planner_realize_v1, - skippy_token_is_eog, skippy_tokenize, skippy_trim_session, skippy_verify_tokens, - skippy_verify_tokens_frame_sampled, skippy_write_gguf_from_parts, + skippy_session_rerank, skippy_session_reset, skippy_session_restore_prefix, + skippy_session_sample_current, skippy_session_save_prefix, skippy_session_sequence_id, + skippy_session_set_position, skippy_session_signal_window, skippy_slice_plan_add_layer_range, + skippy_slice_plan_create, skippy_slice_plan_free, skippy_stage_plan_describe_v1, + skippy_stage_plan_free, skippy_stage_plan_profile_at_v1, + skippy_stage_plan_resident_tensor_at_v1, skippy_stage_plan_state_at_v1, + skippy_stage_plan_string_v1, skippy_stage_plan_validate_chain_v1, + skippy_stage_plan_value_at_v1, skippy_stage_planner_create_v1, skippy_stage_planner_free, + skippy_stage_planner_realize_v1, skippy_token_is_eog, skippy_tokenize, skippy_trim_session, + skippy_verify_tokens, skippy_verify_tokens_frame_sampled, skippy_write_gguf_from_parts, skippy_write_gguf_metadata_from_parts, skippy_write_slice_gguf, }; diff --git a/crates/skippy-ffi/src/multimodal.rs b/crates/skippy-ffi/src/multimodal.rs index 2651dc46c1..0a3f615b1f 100644 --- a/crates/skippy-ffi/src/multimodal.rs +++ b/crates/skippy-ffi/src/multimodal.rs @@ -22,6 +22,48 @@ pub struct MtmdHelperVideo { _private: [u8; 0], } +#[repr(C)] +pub struct MtmdHelperGenAudio { + _private: [u8; 0], +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(i32)] +pub enum MtmdGenAudioType { + None = 0, + Qwen3Tts = 1, + PocketTts = 2, +} + +#[repr(C)] +#[derive(Debug, Clone, Copy)] +pub struct MtmdGenAudioInfo { + pub audio_type: MtmdGenAudioType, + pub sample_rate: i32, + pub model_variant: *const c_char, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(i32)] +pub enum MtmdHelperGenAudioOutputType { + Pcm = 0, + Wav = 1, +} + +#[repr(C)] +#[derive(Debug, Clone, Copy)] +pub struct MtmdHelperGenAudioInput { + pub seq_id: i32, + pub prompt: *const c_char, + pub prompt_len: usize, + pub speaker_ref: *mut MtmdBitmap, + pub lang: *const c_char, + pub top_k: i32, + pub top_p: f32, + pub seed: u32, + pub out_type: MtmdHelperGenAudioOutputType, +} + /// Mirrors llama.cpp's `mtmd_helper_bitmap_wrapper`: the decoded bitmap plus /// an optional video context (non-null only for video inputs, which own the /// frame storage the bitmap points into). diff --git a/crates/skippy-ffi/src/static_bindings.rs b/crates/skippy-ffi/src/static_bindings.rs index a40b48ac0c..ca01fe139d 100644 --- a/crates/skippy-ffi/src/static_bindings.rs +++ b/crates/skippy-ffi/src/static_bindings.rs @@ -4,14 +4,22 @@ use crate::{ ActivationBoundaryDesc, ActivationDesc, BackendDevice, Error, GenerationSignalWindow, IterationRequest, KvPageDesc, LlamaLogCallback, LlamaModelQuantizeParams, Model, ModelInfo, ModelTensorSourceV1, MtmdBitmap, MtmdContext, MtmdContextParams, MtmdDecoderPos, - MtmdHelperBitmapWrapper, MtmdHelperInitOpt, MtmdHelperVideo, MtmdInputChunkType, - MtmdInputChunks, MtmdInputText, NativeMtpDraft, NgramCache, Opaque, RuntimeConfig, - SamplingConfig, Session, SlicePlan, StagePlan, StagePlanDescV1, StagePlanProfileDescV1, - StagePlanStateDescV1, StagePlanStringRefV1, StagePlanValueDescV1, StagePlanValueKind, - StagePlanner, StagePlannerConfigV1, Status, TensorInfo, TokenSignal, + MtmdGenAudioInfo, MtmdHelperBitmapWrapper, MtmdHelperGenAudio, MtmdHelperGenAudioInput, + MtmdHelperInitOpt, MtmdHelperVideo, MtmdInputChunkType, MtmdInputChunks, MtmdInputText, + NativeMtpDraft, NgramCache, Opaque, RuntimeConfig, SamplingConfig, Session, SlicePlan, + StagePlan, StagePlanDescV1, StagePlanProfileDescV1, StagePlanStateDescV1, StagePlanStringRefV1, + StagePlanValueDescV1, StagePlanValueKind, StagePlanner, StagePlannerConfigV1, Status, + TensorInfo, TokenSignal, WorkloadInfoV1, }; unsafe extern "C" { + /// Borrow a token embedding; negative indices count from the last output. + /// Invalid output indices return null. + pub fn llama_get_embeddings_ith(ctx: *mut Opaque, index: i32) -> *mut f32; + + /// Select embedding output instead of logits for subsequent context evaluation. + pub fn llama_set_embeddings(ctx: *mut Opaque, embeddings: bool); + pub fn llama_log_set(log_callback: LlamaLogCallback, user_data: *mut c_void); pub fn ggml_log_set(log_callback: LlamaLogCallback, user_data: *mut c_void); @@ -123,6 +131,13 @@ unsafe extern "C" { out_desc: *mut ActivationBoundaryDesc, ) -> bool; + /// Query an opened model's workload class, pooling, and full-model constraints. + pub fn skippy_model_workload_info_v1( + model: *const Model, + out_info: *mut WorkloadInfoV1, + out_error: *mut *mut Error, + ) -> Status; + pub fn skippy_session_create( model: *mut Model, out_session: *mut *mut Session, @@ -177,6 +192,36 @@ unsafe extern "C" { out_error: *mut *mut Error, ) -> Status; + /// Compute one normalized token-input embedding into a caller-owned buffer. + pub fn skippy_session_embed( + session: *mut Session, + token_ids: *const i32, + token_count: usize, + output: *mut f32, + output_capacity: usize, + out_dimensions: *mut usize, + out_error: *mut *mut Error, + ) -> Status; + + /// Score a NUL-terminated query/document pair and report its token usage. + pub fn skippy_session_rerank( + session: *mut Session, + query: *const c_char, + document: *const c_char, + out_score: *mut f32, + out_token_count: *mut usize, + out_error: *mut *mut Error, + ) -> Status; + + /// Encode source tokens and return the encoder-decoder model's first decoder token. + pub fn skippy_session_encode_prompt( + session: *mut Session, + token_ids: *const i32, + token_count: usize, + out_decoder_start_token: *mut i32, + out_error: *mut *mut Error, + ) -> Status; + pub fn skippy_session_reset(session: *mut Session, out_error: *mut *mut Error) -> Status; pub fn skippy_session_free(session: *mut Session, out_error: *mut *mut Error) -> Status; @@ -711,6 +756,49 @@ unsafe extern "C" { pub fn mtmd_context_params_default() -> MtmdContextParams; + /// Inspect the projector's audio-generation requirements and output metadata. + pub fn mtmd_gen_audio_get_info(ctx: *const MtmdContext) -> MtmdGenAudioInfo; + + /// Allocate an audio generator borrowing the supplied llama and projector contexts. + pub fn mtmd_helper_gen_audio_init( + lctx: *mut Opaque, + mctx: *mut MtmdContext, + ) -> *mut MtmdHelperGenAudio; + + /// Release the generator allocated by `mtmd_helper_gen_audio_init`. + pub fn mtmd_helper_gen_audio_free(ctx: *mut MtmdHelperGenAudio); + + /// Clear generation state and invalidate previously borrowed audio output. + pub fn mtmd_helper_gen_audio_reset(ctx: *mut MtmdHelperGenAudio); + + /// Configure the speech prompt, sampling controls, and output encoding. + pub fn mtmd_helper_gen_audio_set_input( + ctx: *mut MtmdHelperGenAudio, + input: *const MtmdHelperGenAudioInput, + ) -> i32; + + /// Process at most `n_batch` prompt tokens; return remaining tokens, zero, or an error. + pub fn mtmd_helper_gen_audio_step_prompt(ctx: *mut MtmdHelperGenAudio, n_batch: i32) -> i32; + + /// Generate one frame after prompt completion and report end-of-speech. + /// The returned hidden state is borrowed until the next generation step or reset. + pub fn mtmd_helper_gen_audio_step_gen( + ctx: *mut MtmdHelperGenAudio, + sampled: i32, + h_state_in: *const f32, + h_state_out: *mut *const f32, + out_stop: *mut bool, + ) -> i32; + + /// Borrow encoded audio until the next output query or reset, with sample metadata. + pub fn mtmd_helper_gen_audio_get_output( + ctx: *mut MtmdHelperGenAudio, + out_sample_rate: *mut i32, + out_data: *mut *const c_char, + out_data_len: *mut usize, + out_n_samples: *mut i64, + ) -> i32; + pub fn mtmd_init_from_file( mmproj_fname: *const c_char, text_model: *const Opaque, diff --git a/crates/skippy-ffi/src/tests.rs b/crates/skippy-ffi/src/tests.rs index d9e87260f6..69be0f416c 100644 --- a/crates/skippy-ffi/src/tests.rs +++ b/crates/skippy-ffi/src/tests.rs @@ -4,7 +4,8 @@ use crate::{ ABI_VERSION_MAJOR, ABI_VERSION_MINOR, ABI_VERSION_PATCH, AbiVersion, ActivationBoundaryDesc, StagePlanDescV1, StagePlanProfileDescV1, StagePlanStateDescV1, StagePlanStateKind, StagePlanStringRefV1, StagePlanValueDescV1, StagePlannerConfigV1, StagePlannerProfileV1, - StagePlannerTensorV1, runtime_abi_supported, + StagePlannerTensorV1, WORKLOAD_INFO_V1_ABI_VERSION, WorkloadInfoV1, WorkloadKind, + WorkloadPooling, runtime_abi_supported, }; #[cfg(target_pointer_width = "64")] @@ -24,6 +25,69 @@ const fn version(major: u32, minor: u32, patch: u32) -> AbiVersion { } } +#[test] +/// Runtime-event families must never masquerade as full-model workload support. +fn workload_feature_is_distinct_from_runtime_event_features() { + let events = crate::FEATURE_RUNTIME_EVENT_REPORTER + | crate::FEATURE_MODEL_LOAD_EVENTS_V2 + | crate::FEATURE_KV_EVENTS + | crate::FEATURE_DEVICE_EVENTS + | crate::FEATURE_DIAGNOSTIC_EVENTS + | crate::FEATURE_UNLOAD_EVENTS; + assert_eq!(crate::FEATURE_NON_CHAT_WORKLOADS, 1_u64 << 37); + assert_eq!(events & crate::FEATURE_NON_CHAT_WORKLOADS, 0); +} + +/// The linked native library must advertise the same independent feature bits. +#[test] +#[cfg(not(feature = "dynamic-runtime"))] +fn native_workload_feature_preserves_runtime_event_features() { + // SAFETY: This process-global ABI query takes no pointers or mutable state. + let features = unsafe { crate::skippy_abi_features() }; + for expected in [ + crate::FEATURE_NON_CHAT_WORKLOADS, + crate::FEATURE_RUNTIME_EVENT_REPORTER, + crate::FEATURE_MODEL_LOAD_EVENTS_V2, + crate::FEATURE_KV_EVENTS, + crate::FEATURE_DEVICE_EVENTS, + crate::FEATURE_DIAGNOSTIC_EVENTS, + crate::FEATURE_UNLOAD_EVENTS, + ] { + assert_ne!( + features & expected, + 0, + "native feature {expected:#x} is absent" + ); + } +} + +#[test] +/// Pin the C descriptor's field offsets, including booleans and reserved padding. +fn workload_descriptor_matches_native_layout_and_discriminants() { + assert_eq!(WORKLOAD_INFO_V1_ABI_VERSION, 1); + assert_eq!(size_of::(), 28); + assert_eq!(offset_of!(WorkloadInfoV1, abi_version), 0); + assert_eq!(offset_of!(WorkloadInfoV1, struct_size), 4); + assert_eq!(offset_of!(WorkloadInfoV1, kind), 8); + assert_eq!(offset_of!(WorkloadInfoV1, pooling), 12); + assert_eq!(offset_of!(WorkloadInfoV1, output_dimensions), 16); + assert_eq!(offset_of!(WorkloadInfoV1, classifier_outputs), 20); + assert_eq!(offset_of!(WorkloadInfoV1, has_encoder), 24); + assert_eq!(offset_of!(WorkloadInfoV1, has_decoder), 25); + assert_eq!(offset_of!(WorkloadInfoV1, full_model_only), 26); + assert_eq!(offset_of!(WorkloadInfoV1, reserved0), 27); + assert_eq!(WorkloadKind::CausalGeneration as i32, 0); + assert_eq!(WorkloadKind::Embedding as i32, 1); + assert_eq!(WorkloadKind::Rerank as i32, 2); + assert_eq!(WorkloadKind::EncoderDecoder as i32, 3); + assert_eq!(WorkloadPooling::Unspecified as i32, -1); + assert_eq!(WorkloadPooling::None as i32, 0); + assert_eq!(WorkloadPooling::Mean as i32, 1); + assert_eq!(WorkloadPooling::Cls as i32, 2); + assert_eq!(WorkloadPooling::Last as i32, 3); + assert_eq!(WorkloadPooling::Rank as i32, 4); +} + #[test] fn accepts_current_patch_runtime() { assert!(runtime_abi_supported(version( diff --git a/crates/skippy-quantize/src/compose_mtp.rs b/crates/skippy-quantize/src/compose_mtp.rs index cfc637a0cb..d14ae557f9 100644 --- a/crates/skippy-quantize/src/compose_mtp.rs +++ b/crates/skippy-quantize/src/compose_mtp.rs @@ -235,44 +235,34 @@ fn per_layer_suffix(key: &str) -> Option<&'static str> { fn extend_array_kv(kv: &mut GgufKv, mtp_kv: &[GgufKv], layer_count: usize) -> Result<()> { match kv { - GgufKv::ArrayU32 { key, value } => { - if value.len() == layer_count { - let mtp_value = mtp_layer_integer(mtp_kv, key); - let fallback = u64::from(value[value.len() - 1]); - value.push( - u32::try_from(mtp_value.unwrap_or(fallback)) - .context("per-layer array entry overflows uint32")?, - ); - } + GgufKv::ArrayU32 { key, value } if value.len() == layer_count && !value.is_empty() => { + let mtp_value = mtp_layer_integer(mtp_kv, key); + let fallback = u64::from(value[value.len() - 1]); + value.push( + u32::try_from(mtp_value.unwrap_or(fallback)) + .context("per-layer array entry overflows uint32")?, + ); } - GgufKv::ArrayI32 { key, value } => { - if value.len() == layer_count { - let mtp_value = mtp_layer_integer(mtp_kv, key); - let fallback = u64::try_from(i64::from(value[value.len() - 1])) - .context("negative per-layer array entry")?; - value.push( - i32::try_from(mtp_value.unwrap_or(fallback)) - .context("per-layer array entry overflows int32")?, - ); - } + GgufKv::ArrayI32 { key, value } if value.len() == layer_count && !value.is_empty() => { + let mtp_value = mtp_layer_integer(mtp_kv, key); + let fallback = u64::try_from(i64::from(value[value.len() - 1])) + .context("negative per-layer array entry")?; + value.push( + i32::try_from(mtp_value.unwrap_or(fallback)) + .context("per-layer array entry overflows int32")?, + ); } - GgufKv::ArrayF32 { value, .. } => { - if value.len() == layer_count && !value.is_empty() { - let last = value[value.len() - 1]; - value.push(last); - } + GgufKv::ArrayF32 { value, .. } if value.len() == layer_count && !value.is_empty() => { + let last = value[value.len() - 1]; + value.push(last); } - GgufKv::ArrayBool { value, .. } => { - if value.len() == layer_count && !value.is_empty() { - let last = value[value.len() - 1]; - value.push(last); - } + GgufKv::ArrayBool { value, .. } if value.len() == layer_count && !value.is_empty() => { + let last = value[value.len() - 1]; + value.push(last); } - GgufKv::ArrayString { value, .. } => { - if value.len() == layer_count && !value.is_empty() { - let last = value[value.len() - 1].clone(); - value.push(last); - } + GgufKv::ArrayString { value, .. } if value.len() == layer_count && !value.is_empty() => { + let last = value[value.len() - 1].clone(); + value.push(last); } // Typed-array variants cover only u32/i32/f32/bool/string; other // element widths (e.g. the u16 `attention.head_count` the Nemotron diff --git a/crates/skippy-runtime/src/activation.rs b/crates/skippy-runtime/src/activation.rs index 69dd78f117..2a0941a4c5 100644 --- a/crates/skippy-runtime/src/activation.rs +++ b/crates/skippy-runtime/src/activation.rs @@ -1113,6 +1113,10 @@ fn validate_serial_decode_request(request: &IterationBatchRequest<'_>) -> Result Ok(()) } +#[cfg(test)] +#[path = "activation/sampling_tests.rs"] +mod sampling_tests; + #[cfg(test)] mod tests { use super::{ diff --git a/crates/skippy-runtime/src/activation/sampling_tests.rs b/crates/skippy-runtime/src/activation/sampling_tests.rs new file mode 100644 index 0000000000..962bea81e3 --- /dev/null +++ b/crates/skippy-runtime/src/activation/sampling_tests.rs @@ -0,0 +1,46 @@ +//! Native output-buffer contracts for sampled prefill with no output head. + +use crate::{ModelInfo, RuntimeConfig, StageModel, TensorRole}; +use anyhow::{Context as _, Result}; + +/// The raw wrapper starts its token buffer at zero; native prefill must replace +/// it with the no-token sentinel on every successful output-disabled call. +#[test] +fn output_disabled_prefill_clears_sampled_token_when_fixture_is_set() -> Result<()> { + let Some(path) = std::env::var_os("SKIPPY_CORRECTNESS_MODEL") else { + return Ok(()); + }; + let _native_log_guard = crate::logging::native_log_test_guard(); + let info = ModelInfo::open(&path)?; + let layer_end = info + .tensors()? + .iter() + .filter(|tensor| tensor.role == TensorRole::Layer) + .filter_map(|tensor| tensor.layer_index) + .max() + .context("correctness fixture has no transformer layers")? + + 1; + let model = StageModel::open( + &path, + &RuntimeConfig { + layer_end, + ctx_size: 256, + n_gpu_layers: 0, + include_output: false, + filter_tensors_on_load: false, + ..RuntimeConfig::default() + }, + )?; + let tokens = model.tokenize("A short native prefill regression.", true)?; + let mut session = model.create_session()?; + for positions in [Vec::new(), (0..i32::try_from(tokens.len())?).collect()] { + session.reset()?; + let (token, _, _) = + session.prefill_chunk_frame_sampled_raw(&tokens, &positions, None, None, 0)?; + assert_eq!( + token, -1, + "an output-disabled prefill must never leak a token" + ); + } + Ok(()) +} diff --git a/crates/skippy-runtime/src/lib.rs b/crates/skippy-runtime/src/lib.rs index d90966144d..f8543ae3ee 100644 --- a/crates/skippy-runtime/src/lib.rs +++ b/crates/skippy-runtime/src/lib.rs @@ -44,7 +44,8 @@ pub use logging::{ set_filtered_native_logs_enabled, suppress_native_logs, unregister_filtered_native_logs, write_native_log_note, }; -pub use native::{StageModel, StageModelReader}; +pub use media::{SpeechAudio, SpeechOutputFormat, SpeechSynthesisConfig}; +pub use native::{ModelWorkload, PoolingType, StageModel, StageModelReader, WorkloadInfo}; pub use native_mtp::NativeMtpDraft; pub use native_test_evidence::write_evidence_marker; pub use ngram::{Cache as NgramCache, NGRAM_CACHE_MAX_NGRAM}; diff --git a/crates/skippy-runtime/src/media.rs b/crates/skippy-runtime/src/media.rs index de634a1111..4cc7b7b4f2 100644 --- a/crates/skippy-runtime/src/media.rs +++ b/crates/skippy-runtime/src/media.rs @@ -13,6 +13,48 @@ use crate::{ SamplingConfig, }; +/// Audio encoding returned by the native speech generator. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SpeechOutputFormat { + Wav, + PcmS16Le, +} + +/// Full-model speech inputs and deterministic sampling controls. +#[derive(Debug, Clone, PartialEq)] +pub struct SpeechSynthesisConfig { + pub prompt: String, + pub language: Option, + pub top_k: i32, + pub top_p: f32, + pub seed: u32, + pub output_format: SpeechOutputFormat, + pub max_frames: usize, +} + +/// Complete generated audio and its native frame count. Reaching the configured +/// frame limit returns an error instead of this response. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SpeechAudio { + pub bytes: Vec, + pub sample_rate: u32, + pub sample_count: u64, + pub generated_frames: usize, +} + +/// Restore the session's generation mode on every speech exit, including +/// failures before the external-decode guard can be acquired. +struct SpeechEmbeddingsGuard(*mut skippy_ffi::Opaque); + +impl Drop for SpeechEmbeddingsGuard { + /// Restore logits output even when speech setup or generation exits with an error. + fn drop(&mut self) { + // SAFETY: the borrowed StageSession outlives this guard and owns the + // context; speech generation holds exclusive access to the session. + unsafe { skippy_ffi::llama_set_embeddings(self.0, false) }; + } +} + pub(crate) struct MediaProjector { pub(crate) raw: *mut skippy_ffi::MtmdContext, marker: String, @@ -26,95 +68,8 @@ type MediaFrameEval = ( Vec, ); -fn aggregate_media_chunk_outputs(chunks: &[MediaPrefillChunkFrame]) -> Result { - let first = chunks - .first() - .ok_or_else(|| anyhow!("multimodal prefill produced no activation output"))?; - let mut desc = first.output.desc; - let mut token_count = 0usize; - let mut common_flags = desc.flags; - - for (index, chunk) in chunks.iter().enumerate() { - let frame = &chunk.output; - if desc.version != frame.desc.version - || desc.dtype != frame.desc.dtype - || desc.layout != frame.desc.layout - || desc.producer_stage_index != frame.desc.producer_stage_index - || desc.layer_start != frame.desc.layer_start - || desc.layer_end != frame.desc.layer_end - || desc.sequence_count != frame.desc.sequence_count - { - return Err(anyhow!( - "multimodal chunk {index} produced incompatible activation descriptor" - )); - } - token_count = token_count - .checked_add(chunk.token_count) - .context("multimodal activation token count overflow")?; - common_flags &= frame.desc.flags; - } - - let differing_flags = chunks - .iter() - .fold(0_u64, |flags, chunk| flags | chunk.output.desc.flags) - & !common_flags; - if differing_flags & !skippy_ffi::ACTIVATION_FLAG_INKLING_MTP_EMBD != 0 { - return Err(anyhow!( - "multimodal chunks produced incompatible activation sideband flags {differing_flags:#x}" - )); - } - - let mut payload = Vec::new(); - if differing_flags == skippy_ffi::ACTIVATION_FLAG_INKLING_MTP_EMBD { - let base_chunk = chunks - .iter() - .find(|chunk| { - chunk.output.desc.flags & skippy_ffi::ACTIVATION_FLAG_INKLING_MTP_EMBD == 0 - }) - .ok_or_else(|| anyhow!("multimodal Inkling output has no base activation chunk"))?; - let bytes_per_token = base_chunk - .output - .payload - .len() - .checked_div(base_chunk.token_count) - .filter(|_| base_chunk.output.payload.len() % base_chunk.token_count == 0) - .ok_or_else(|| anyhow!("multimodal base activation payload is not token-aligned"))?; - for (index, chunk) in chunks.iter().enumerate() { - let hidden_bytes = bytes_per_token - .checked_mul(chunk.token_count) - .context("multimodal base activation byte count overflow")?; - let hidden = chunk.output.payload.get(..hidden_bytes).ok_or_else(|| { - anyhow!("multimodal chunk {index} is smaller than its base activation payload") - })?; - payload.extend_from_slice(hidden); - } - desc.flags = common_flags; - } else if common_flags & skippy_ffi::ACTIVATION_FLAG_INKLING_MTP_EMBD != 0 { - let mut hidden_planes = Vec::new(); - let mut mtp_planes = Vec::new(); - for (index, chunk) in chunks.iter().enumerate() { - if chunk.output.payload.len() % 2 != 0 { - return Err(anyhow!( - "multimodal Inkling chunk {index} sideband payload is not evenly split" - )); - } - let plane_bytes = chunk.output.payload.len() / 2; - hidden_planes.extend_from_slice(&chunk.output.payload[..plane_bytes]); - mtp_planes.extend_from_slice(&chunk.output.payload[plane_bytes..]); - } - payload = hidden_planes; - payload.extend_from_slice(&mtp_planes); - } else { - for chunk in chunks { - payload.extend_from_slice(&chunk.output.payload); - } - } - - desc.token_count = u32::try_from(token_count).context("multimodal token count exceeds u32")?; - desc.payload_bytes = - u64::try_from(payload.len()).context("multimodal activation payload length exceeds u64")?; - Ok(ActivationFrame { desc, payload }) -} +mod chunk_aggregation; +use chunk_aggregation::aggregate_media_chunk_outputs; // The experimental C ABI owns synchronization internally for model/session use. // Rust stage-server access is additionally serialized behind a Mutex. @@ -200,6 +155,199 @@ impl StageModel { self.media.is_some() } + /// Report whether the configured projector supports native audio generation. + pub fn supports_speech_synthesis(&self) -> bool { + self.media.as_ref().is_some_and(|projector| { + let info = unsafe { skippy_ffi::mtmd_gen_audio_get_info(projector.raw) }; + info.audio_type != skippy_ffi::MtmdGenAudioType::None + }) + } + + /// Generate bounded audio while restoring the session's normal decode mode + /// on success, cancellation, and native failure. Sessions remain exclusive. + pub fn synthesize_speech( + &self, + session: &mut StageSession, + config: &SpeechSynthesisConfig, + cancellation_requested: impl Fn() -> bool, + ) -> Result { + let projector = self + .media + .as_ref() + .ok_or_else(|| anyhow!("speech synthesis requires a configured projector"))?; + let info = unsafe { skippy_ffi::mtmd_gen_audio_get_info(projector.raw) }; + if info.audio_type == skippy_ffi::MtmdGenAudioType::None { + return Err(anyhow!( + "configured projector does not support speech synthesis" + )); + } + if config.prompt.is_empty() || config.max_frames == 0 { + return Err(anyhow!( + "speech prompt and max_frames must not be empty or zero" + )); + } + let prompt = CString::new(config.prompt.as_bytes()) + .context("speech prompt contains an interior NUL byte")?; + let language = config + .language + .as_deref() + .map(CString::new) + .transpose() + .context("speech language contains an interior NUL byte")?; + let lctx = unsafe { skippy_ffi::skippy_session_llama_context(session.raw) }; + if lctx.is_null() { + return Err(anyhow!("speech session did not expose a llama context")); + } + + struct AudioGenerator(*mut skippy_ffi::MtmdHelperGenAudio); + impl Drop for AudioGenerator { + /// Release the native generator without taking ownership of its contexts. + fn drop(&mut self) { + if !self.0.is_null() { + unsafe { skippy_ffi::mtmd_helper_gen_audio_free(self.0) }; + } + } + } + struct ExternalDecodeGuard(*mut skippy_ffi::Session); + impl Drop for ExternalDecodeGuard { + /// End speech's external-decode scope and free any native cleanup error. + fn drop(&mut self) { + let mut error = ptr::null_mut(); + unsafe { + let _ = skippy_ffi::skippy_session_end_external_decode(self.0, &mut error); + } + free_error(error); + } + } + session.reset()?; + unsafe { skippy_ffi::llama_set_embeddings(lctx, true) }; + let _embeddings_mode = SpeechEmbeddingsGuard(lctx); + let mut guard_error = ptr::null_mut(); + let status = unsafe { + skippy_ffi::skippy_session_begin_external_decode(session.raw, &mut guard_error) + }; + ensure_ok(status, guard_error)?; + let _external_decode = ExternalDecodeGuard(session.raw); + let generator = + AudioGenerator(unsafe { skippy_ffi::mtmd_helper_gen_audio_init(lctx, projector.raw) }); + if generator.0.is_null() { + return Err(anyhow!("failed to initialize speech synthesis pipeline")); + } + let output_type = match config.output_format { + SpeechOutputFormat::Wav => skippy_ffi::MtmdHelperGenAudioOutputType::Wav, + SpeechOutputFormat::PcmS16Le => skippy_ffi::MtmdHelperGenAudioOutputType::Pcm, + }; + let input = skippy_ffi::MtmdHelperGenAudioInput { + seq_id: session.native_sequence_id()?, + prompt: prompt.as_ptr(), + prompt_len: config.prompt.len(), + speaker_ref: ptr::null_mut(), + lang: language + .as_ref() + .map_or(ptr::null(), |value| value.as_ptr()), + top_k: config.top_k, + top_p: config.top_p, + seed: config.seed, + out_type: output_type, + }; + if unsafe { skippy_ffi::mtmd_helper_gen_audio_set_input(generator.0, &input) } != 0 { + return Err(anyhow!("speech synthesis rejected the input")); + } + let batch_size = + i32::try_from(session.batch_size()?).context("speech batch size exceeds i32")?; + loop { + if cancellation_requested() { + return Err(anyhow!("speech synthesis cancelled")); + } + let remaining = + unsafe { skippy_ffi::mtmd_helper_gen_audio_step_prompt(generator.0, batch_size) }; + if remaining < 0 { + return Err(anyhow!("speech prompt evaluation failed")); + } + if remaining == 0 { + break; + } + } + + let sampling = SamplingConfig { + enabled: true, + seed: config.seed, + top_k: config.top_k, + top_p: config.top_p, + ..SamplingConfig::default() + }; + let mut sampled = session.sample_current(Some(&sampling))?; + let mut hidden_state = + unsafe { skippy_ffi::llama_get_embeddings_ith(lctx, -1) }.cast_const(); + if hidden_state.is_null() { + return Err(anyhow!("speech backbone did not produce a hidden state")); + } + let mut generated_frames = 0usize; + let mut stopped = false; + while generated_frames < config.max_frames { + if cancellation_requested() { + return Err(anyhow!("speech synthesis cancelled")); + } + let mut next_hidden_state = ptr::null(); + let mut stop = false; + let step = unsafe { + skippy_ffi::mtmd_helper_gen_audio_step_gen( + generator.0, + sampled, + hidden_state, + &mut next_hidden_state, + &mut stop, + ) + }; + if step != 0 { + return Err(anyhow!( + "speech synthesis failed at frame {generated_frames}" + )); + } + if stop || next_hidden_state.is_null() { + stopped = true; + break; + } + generated_frames += 1; + hidden_state = next_hidden_state; + sampled = session.sample_current(Some(&sampling))?; + } + if !stopped { + return Err(anyhow!( + "speech synthesis exceeded the configured {} frame limit", + config.max_frames + )); + } + + let mut sample_rate = 0_i32; + let mut data = ptr::null(); + let mut data_len = 0usize; + let mut sample_count = 0_i64; + let output_status = unsafe { + skippy_ffi::mtmd_helper_gen_audio_get_output( + generator.0, + &mut sample_rate, + &mut data, + &mut data_len, + &mut sample_count, + ) + }; + if output_status != 0 || data.is_null() || data_len == 0 { + return Err(anyhow!("speech synthesis produced no audio")); + } + let native_bytes = unsafe { std::slice::from_raw_parts(data.cast::(), data_len) }; + let bytes = match config.output_format { + SpeechOutputFormat::Wav => native_bytes.to_vec(), + SpeechOutputFormat::PcmS16Le => pcm_f32_to_s16le(native_bytes)?, + }; + Ok(SpeechAudio { + bytes, + sample_rate: u32::try_from(sample_rate).context("invalid speech sample rate")?, + sample_count: u64::try_from(sample_count).context("invalid speech sample count")?, + generated_frames, + }) + } + fn eval_media( &self, session: &mut StageSession, @@ -704,74 +852,56 @@ impl StageModel { } } +/// Validate and quantize native float32 PCM into clamped signed little-endian samples. +fn pcm_f32_to_s16le(bytes: &[u8]) -> Result> { + if !bytes.len().is_multiple_of(std::mem::size_of::()) { + return Err(anyhow!("native PCM payload is not aligned to f32 samples")); + } + let mut output = Vec::with_capacity(bytes.len() / 2); + let (samples, remainder) = bytes.as_chunks::<4>(); + debug_assert!(remainder.is_empty()); + for sample in samples { + let sample = f32::from_ne_bytes(*sample); + let quantized = (sample.clamp(-1.0, 1.0) * f32::from(i16::MAX)).round() as i16; + output.extend_from_slice(&quantized.to_le_bytes()); + } + Ok(output) +} + #[cfg(test)] mod tests { - use super::aggregate_media_chunk_outputs; - use crate::{ - ActivationDesc, ActivationFrame, MediaPrefillChunkFrame, RuntimeActivationDType, - RuntimeActivationLayout, - }; - - fn chunk(token_count: usize, flags: u64, payload: Vec) -> MediaPrefillChunkFrame { - MediaPrefillChunkFrame { - token_count, - tokens: Vec::new(), - positions: Vec::new(), - output: ActivationFrame { - desc: ActivationDesc { - version: 1, - dtype: RuntimeActivationDType::F32, - layout: RuntimeActivationLayout::TokenMajor, - producer_stage_index: 0, - layer_start: 0, - layer_end: 1, - token_count: token_count as u32, - sequence_count: 1, - payload_bytes: payload.len() as u64, - flags, - }, - payload, - }, - } - } + use super::pcm_f32_to_s16le; #[test] - fn mixed_inkling_chunks_aggregate_the_common_hidden_plane() -> anyhow::Result<()> { - let chunks = vec![ - chunk(1, 0, vec![1, 2, 3, 4]), - chunk( - 2, - skippy_ffi::ACTIVATION_FLAG_INKLING_MTP_EMBD, - vec![5, 6, 7, 8, 9, 10, 11, 12, 21, 22, 23, 24, 25, 26, 27, 28], - ), - ]; - - let output = aggregate_media_chunk_outputs(&chunks)?; - - assert_eq!(output.desc.token_count, 3); - assert_eq!(output.desc.flags, 0); - assert_eq!(output.desc.payload_bytes, 12); - assert_eq!(output.payload, vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]); - Ok(()) - } + /// Verify clipping and quantization at signed PCM boundaries. + fn pcm_conversion_clamps_and_quantizes_native_float_samples() { + let samples = [-2.0_f32, -1.0, -0.5, 0.0, 0.5, 1.0, 2.0]; + let bytes = samples + .iter() + .flat_map(|sample| sample.to_ne_bytes()) + .collect::>(); + + let converted = pcm_f32_to_s16le(&bytes).expect("aligned native PCM"); + let (converted_samples, remainder) = converted.as_chunks::<2>(); + assert!(remainder.is_empty()); + let actual = converted_samples + .iter() + .map(|sample| i16::from_le_bytes(*sample)) + .collect::>(); - #[test] - fn uniform_inkling_chunks_aggregate_each_plane_in_token_order() -> anyhow::Result<()> { - let flag = skippy_ffi::ACTIVATION_FLAG_INKLING_MTP_EMBD; - let chunks = vec![ - chunk(1, flag, vec![1, 2, 3, 4, 11, 12, 13, 14]), - chunk(1, flag, vec![5, 6, 7, 8, 15, 16, 17, 18]), - ]; - - let output = aggregate_media_chunk_outputs(&chunks)?; - - assert_eq!(output.desc.token_count, 2); - assert_eq!(output.desc.flags, flag); - assert_eq!(output.desc.payload_bytes, 16); assert_eq!( - output.payload, - vec![1, 2, 3, 4, 5, 6, 7, 8, 11, 12, 13, 14, 15, 16, 17, 18] + actual, + vec![-32_767, -32_767, -16_384, 0, 16_384, 32_767, 32_767] ); - Ok(()) + } + + #[test] + /// Reject PCM payloads without complete float32 samples. + fn pcm_conversion_rejects_misaligned_native_payload() { + assert!(pcm_f32_to_s16le(&[0, 1, 2]).is_err()); } } + +#[cfg(test)] +#[path = "media/speech_session_tests.rs"] +mod speech_session_tests; diff --git a/crates/skippy-runtime/src/media/chunk_aggregation.rs b/crates/skippy-runtime/src/media/chunk_aggregation.rs new file mode 100644 index 0000000000..7f0692b281 --- /dev/null +++ b/crates/skippy-runtime/src/media/chunk_aggregation.rs @@ -0,0 +1,204 @@ +use anyhow::{Context, Result, anyhow}; + +use crate::{ActivationFrame, MediaPrefillChunkFrame}; + +pub(super) fn aggregate_media_chunk_outputs( + chunks: &[MediaPrefillChunkFrame], +) -> Result { + let first = chunks + .first() + .ok_or_else(|| anyhow!("multimodal prefill produced no activation output"))?; + let mut desc = first.output.desc; + let mut token_count = 0usize; + let mut common_flags = desc.flags; + + for (index, chunk) in chunks.iter().enumerate() { + let frame = &chunk.output; + if desc.version != frame.desc.version + || desc.dtype != frame.desc.dtype + || desc.layout != frame.desc.layout + || desc.producer_stage_index != frame.desc.producer_stage_index + || desc.layer_start != frame.desc.layer_start + || desc.layer_end != frame.desc.layer_end + || desc.sequence_count != frame.desc.sequence_count + { + return Err(anyhow!( + "multimodal chunk {index} produced incompatible activation descriptor" + )); + } + token_count = token_count + .checked_add(chunk.token_count) + .context("multimodal activation token count overflow")?; + common_flags &= frame.desc.flags; + } + + let differing_flags = chunks + .iter() + .fold(0_u64, |flags, chunk| flags | chunk.output.desc.flags) + & !common_flags; + if differing_flags & !skippy_ffi::ACTIVATION_FLAG_INKLING_MTP_EMBD != 0 { + return Err(anyhow!( + "multimodal chunks produced incompatible activation sideband flags {differing_flags:#x}" + )); + } + + let mut payload = Vec::new(); + if differing_flags == skippy_ffi::ACTIVATION_FLAG_INKLING_MTP_EMBD { + let base_chunk = chunks + .iter() + .find(|chunk| { + chunk.output.desc.flags & skippy_ffi::ACTIVATION_FLAG_INKLING_MTP_EMBD == 0 + }) + .ok_or_else(|| anyhow!("multimodal Inkling output has no base activation chunk"))?; + let bytes_per_token = base_chunk + .output + .payload + .len() + .checked_div(base_chunk.token_count) + .filter(|_| base_chunk.output.payload.len() % base_chunk.token_count == 0) + .ok_or_else(|| anyhow!("multimodal base activation payload is not token-aligned"))?; + for (index, chunk) in chunks.iter().enumerate() { + let hidden_bytes = bytes_per_token + .checked_mul(chunk.token_count) + .context("multimodal base activation byte count overflow")?; + let hidden = chunk.output.payload.get(..hidden_bytes).ok_or_else(|| { + anyhow!("multimodal chunk {index} is smaller than its base activation payload") + })?; + payload.extend_from_slice(hidden); + } + desc.flags = common_flags; + } else if common_flags & skippy_ffi::ACTIVATION_FLAG_INKLING_MTP_EMBD != 0 { + let mut hidden_planes = Vec::new(); + let mut mtp_planes = Vec::new(); + for (index, chunk) in chunks.iter().enumerate() { + if chunk.output.payload.len() % 2 != 0 { + return Err(anyhow!( + "multimodal Inkling chunk {index} sideband payload is not evenly split" + )); + } + let plane_bytes = chunk.output.payload.len() / 2; + hidden_planes.extend_from_slice(&chunk.output.payload[..plane_bytes]); + mtp_planes.extend_from_slice(&chunk.output.payload[plane_bytes..]); + } + payload = hidden_planes; + payload.extend_from_slice(&mtp_planes); + } else { + for chunk in chunks { + payload.extend_from_slice(&chunk.output.payload); + } + } + + desc.token_count = u32::try_from(token_count).context("multimodal token count exceeds u32")?; + desc.payload_bytes = + u64::try_from(payload.len()).context("multimodal activation payload length exceeds u64")?; + Ok(ActivationFrame { desc, payload }) +} + +#[cfg(test)] +mod tests { + use super::aggregate_media_chunk_outputs; + use crate::{ + ActivationDesc, ActivationFrame, MediaPrefillChunkFrame, RuntimeActivationDType, + RuntimeActivationLayout, + }; + + fn chunk(token_count: usize, flags: u64, payload: Vec) -> MediaPrefillChunkFrame { + MediaPrefillChunkFrame { + token_count, + tokens: Vec::new(), + positions: Vec::new(), + output: ActivationFrame { + desc: ActivationDesc { + version: 1, + dtype: RuntimeActivationDType::F32, + layout: RuntimeActivationLayout::TokenMajor, + producer_stage_index: 0, + layer_start: 0, + layer_end: 1, + token_count: token_count as u32, + sequence_count: 1, + payload_bytes: payload.len() as u64, + flags, + }, + payload, + }, + } + } + + #[test] + fn aggregation_rejects_empty_output() { + assert!(aggregate_media_chunk_outputs(&[]).is_err()); + } + + #[test] + fn aggregation_rejects_different_stage_descriptors() { + let first = chunk(1, 0, vec![1, 2, 3, 4]); + let mut second = chunk(1, 0, vec![5, 6, 7, 8]); + second.output.desc.producer_stage_index = 1; + let error = aggregate_media_chunk_outputs(&[first, second]).unwrap_err(); + assert!( + error + .to_string() + .contains("incompatible activation descriptor") + ); + } + + #[test] + fn only_inkling_sidebands_may_differ_between_chunks() { + for flag in [ + skippy_ffi::ACTIVATION_FLAG_GLM_DSA_TOP_K, + skippy_ffi::ACTIVATION_FLAG_KIMI_K3_RESIDUAL, + ] { + let chunks = [ + chunk(1, 0, vec![1, 2, 3, 4]), + chunk(1, flag, vec![5, 6, 7, 8]), + ]; + let error = aggregate_media_chunk_outputs(&chunks).unwrap_err(); + assert!( + error + .to_string() + .contains("incompatible activation sideband flags") + ); + } + } + + #[test] + fn mixed_inkling_chunks_aggregate_the_common_hidden_plane() -> anyhow::Result<()> { + let chunks = vec![ + chunk(1, 0, vec![1, 2, 3, 4]), + chunk( + 2, + skippy_ffi::ACTIVATION_FLAG_INKLING_MTP_EMBD, + vec![5, 6, 7, 8, 9, 10, 11, 12, 21, 22, 23, 24, 25, 26, 27, 28], + ), + ]; + + let output = aggregate_media_chunk_outputs(&chunks)?; + + assert_eq!(output.desc.token_count, 3); + assert_eq!(output.desc.flags, 0); + assert_eq!(output.desc.payload_bytes, 12); + assert_eq!(output.payload, vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]); + Ok(()) + } + + #[test] + fn uniform_inkling_chunks_aggregate_each_plane_in_token_order() -> anyhow::Result<()> { + let flag = skippy_ffi::ACTIVATION_FLAG_INKLING_MTP_EMBD; + let chunks = vec![ + chunk(1, flag, vec![1, 2, 3, 4, 11, 12, 13, 14]), + chunk(1, flag, vec![5, 6, 7, 8, 15, 16, 17, 18]), + ]; + + let output = aggregate_media_chunk_outputs(&chunks)?; + + assert_eq!(output.desc.token_count, 2); + assert_eq!(output.desc.flags, flag); + assert_eq!(output.desc.payload_bytes, 16); + assert_eq!( + output.payload, + vec![1, 2, 3, 4, 5, 6, 7, 8, 11, 12, 13, 14, 15, 16, 17, 18] + ); + Ok(()) + } +} diff --git a/crates/skippy-runtime/src/media/speech_session_tests.rs b/crates/skippy-runtime/src/media/speech_session_tests.rs new file mode 100644 index 0000000000..74e2c31b71 --- /dev/null +++ b/crates/skippy-runtime/src/media/speech_session_tests.rs @@ -0,0 +1,91 @@ +//! Real-projector lifecycle regressions, opt-in through the workload fixture. + +use super::*; +use crate::{ModelInfo, RuntimeConfig, TensorRole}; + +/// Load the opt-in speech model and projector without a synthetic runtime fallback. +fn speech_fixture() -> Result> { + if std::env::var("SKIPPY_WORKLOAD_CLASS").as_deref() != Ok("speech_synthesis") { + return Ok(None); + } + let path = std::env::var("SKIPPY_WORKLOAD_MODEL").context("speech fixture model")?; + let projector = + std::env::var("SKIPPY_WORKLOAD_PROJECTOR").context("speech fixture projector")?; + let layer_end = ModelInfo::open(&path)? + .tensors()? + .into_iter() + .filter(|tensor| tensor.role == TensorRole::Layer) + .filter_map(|tensor| tensor.layer_index) + .max() + .context("speech fixture layers")? + + 1; + let config = RuntimeConfig { + layer_end, + ctx_size: 2048, + n_batch: Some(2048), + n_ubatch: Some(2048), + projector_path: Some(projector), + projector_use_gpu: Some(false), + kv_offload: Some(false), + op_offload: Some(false), + ..RuntimeConfig::default() + }; + StageModel::open(path, &config).map(Some) +} + +/// Prove a speech session can resume ordinary decoding with a real output row. +fn assert_generation_reusable(model: &StageModel, session: &mut StageSession) -> Result<()> { + session.reset()?; + let tokens = model.tokenize("The mesh is ready.", true)?; + let (last, prefix) = tokens.split_last().context("generation fixture tokens")?; + session.prefill_chunked(prefix)?; + // Plain prefill deliberately produces no logits. Decode the final token + // with an output row so this checks real generation, not a sampler fallback. + let token = session.decode_step(*last)?; + assert!(token >= 0, "normal generation must produce a valid token"); + assert_eq!(session.sample_current(None)?, token); + Ok(()) +} + +#[test] +/// Verify all speech exit paths restore ordinary session decoding. +fn speech_success_cancellation_and_native_failure_leave_session_reusable() -> Result<()> { + let Some(model) = speech_fixture()? else { + return Ok(()); + }; + let mut session = model.create_session()?; + let mut config = SpeechSynthesisConfig { + prompt: "Hello.".into(), + language: None, + top_k: 1, + top_p: 1.0, + seed: 42, + output_format: SpeechOutputFormat::Wav, + max_frames: 2, + }; + let cancelled = model + .synthesize_speech(&mut session, &config, || true) + .unwrap_err(); + assert!(cancelled.to_string().contains("cancelled"), "{cancelled:#}"); + assert_generation_reusable(&model, &mut session)?; + config.language = Some("not-a-supported-language".into()); + let rejected = model + .synthesize_speech(&mut session, &config, || false) + .unwrap_err(); + assert!( + rejected.to_string().contains("rejected the input"), + "{rejected:#}" + ); + assert_generation_reusable(&model, &mut session)?; + config.language = None; + let capped = model + .synthesize_speech(&mut session, &config, || false) + .unwrap_err(); + assert!(capped.to_string().contains("frame limit"), "{capped:#}"); + assert_generation_reusable(&model, &mut session)?; + config.max_frames = 512; + let audio = model.synthesize_speech(&mut session, &config, || false)?; + assert!(audio.sample_count > 0 && audio.bytes.len() > 44); + assert_generation_reusable(&model, &mut session)?; + Ok(()) +} diff --git a/crates/skippy-runtime/src/native.rs b/crates/skippy-runtime/src/native.rs index fc5c7fbc52..e6fd8cdcb9 100644 --- a/crates/skippy-runtime/src/native.rs +++ b/crates/skippy-runtime/src/native.rs @@ -18,6 +18,73 @@ use crate::{ RuntimeEvent, Status, }; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ModelWorkload { + CausalGeneration, + Embedding, + Rerank, + EncoderDecoder, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PoolingType { + Unspecified, + None, + Mean, + Cls, + Last, + Rank, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct WorkloadInfo { + pub kind: ModelWorkload, + pub pooling: PoolingType, + pub output_dimensions: u32, + pub classifier_outputs: u32, + pub has_encoder: bool, + pub has_decoder: bool, + pub full_model_only: bool, +} + +impl TryFrom for WorkloadInfo { + type Error = anyhow::Error; + + /// Validate the native descriptor layout and translate supported workload and pooling values. + fn try_from(raw: skippy_ffi::WorkloadInfoV1) -> Result { + if raw.abi_version != skippy_ffi::WORKLOAD_INFO_V1_ABI_VERSION + || raw.struct_size != std::mem::size_of::() as u32 + { + return Err(anyhow!( + "native workload descriptor uses an incompatible ABI" + )); + } + let kind = match raw.kind { + skippy_ffi::WorkloadKind::CausalGeneration => ModelWorkload::CausalGeneration, + skippy_ffi::WorkloadKind::Embedding => ModelWorkload::Embedding, + skippy_ffi::WorkloadKind::Rerank => ModelWorkload::Rerank, + skippy_ffi::WorkloadKind::EncoderDecoder => ModelWorkload::EncoderDecoder, + }; + let pooling = match raw.pooling { + skippy_ffi::WorkloadPooling::Unspecified => PoolingType::Unspecified, + skippy_ffi::WorkloadPooling::None => PoolingType::None, + skippy_ffi::WorkloadPooling::Mean => PoolingType::Mean, + skippy_ffi::WorkloadPooling::Cls => PoolingType::Cls, + skippy_ffi::WorkloadPooling::Last => PoolingType::Last, + skippy_ffi::WorkloadPooling::Rank => PoolingType::Rank, + }; + Ok(Self { + kind, + pooling, + output_dimensions: raw.output_dimensions, + classifier_outputs: raw.classifier_outputs, + has_encoder: raw.has_encoder, + has_decoder: raw.has_decoder, + full_model_only: raw.full_model_only, + }) + } +} + pub struct StageModel { inner: Arc, pub(crate) media: Option, @@ -106,6 +173,17 @@ impl StageModel { present.then(|| raw.into()) } + /// Read the loaded model's ABI-validated workload, pooling, and output dimensions. + pub fn workload_info(&self) -> Result { + let mut raw = skippy_ffi::WorkloadInfoV1::default(); + let mut error = ptr::null_mut(); + let status = unsafe { + skippy_ffi::skippy_model_workload_info_v1(self.inner.raw, &mut raw, &mut error) + }; + ensure_ok(status, error)?; + raw.try_into() + } + fn from_opened_raw( raw: *mut RawModel, config: &RuntimeConfig, @@ -992,11 +1070,68 @@ impl Drop for StageModel { #[cfg(test)] mod output_capacity_tests { use super::{ - ModelStateKind, OPTIMISTIC_OUTPUT_HEADROOM, capability_from_state_probes, - classify_model_state, optimistic_chat_metadata_capacity, optimistic_chat_parse_capacity, - optimistic_chat_prompt_capacity, optimistic_token_capacity, + ModelStateKind, ModelWorkload, OPTIMISTIC_OUTPUT_HEADROOM, PoolingType, WorkloadInfo, + capability_from_state_probes, classify_model_state, optimistic_chat_metadata_capacity, + optimistic_chat_parse_capacity, optimistic_chat_prompt_capacity, optimistic_token_capacity, }; + #[test] + /// Cover all supported native workload and pooling discriminants. + fn workload_descriptor_converts_all_native_classes_and_pooling_modes() { + let cases = [ + ( + skippy_ffi::WorkloadKind::CausalGeneration, + ModelWorkload::CausalGeneration, + ), + ( + skippy_ffi::WorkloadKind::Embedding, + ModelWorkload::Embedding, + ), + (skippy_ffi::WorkloadKind::Rerank, ModelWorkload::Rerank), + ( + skippy_ffi::WorkloadKind::EncoderDecoder, + ModelWorkload::EncoderDecoder, + ), + ]; + for (raw_kind, expected_kind) in cases { + let converted = WorkloadInfo::try_from(skippy_ffi::WorkloadInfoV1 { + kind: raw_kind, + pooling: skippy_ffi::WorkloadPooling::Mean, + output_dimensions: 768, + classifier_outputs: 2, + has_encoder: true, + has_decoder: false, + full_model_only: true, + ..Default::default() + }) + .expect("valid workload descriptor"); + + assert_eq!(converted.kind, expected_kind); + assert_eq!(converted.pooling, PoolingType::Mean); + assert_eq!(converted.output_dimensions, 768); + assert_eq!(converted.classifier_outputs, 2); + assert!(converted.has_encoder); + assert!(!converted.has_decoder); + assert!(converted.full_model_only); + } + } + + #[test] + /// Reject incompatible native descriptor versions and sizes. + fn workload_descriptor_rejects_incompatible_layout_versions() { + let invalid_version = skippy_ffi::WorkloadInfoV1 { + abi_version: skippy_ffi::WORKLOAD_INFO_V1_ABI_VERSION + 1, + ..Default::default() + }; + assert!(WorkloadInfo::try_from(invalid_version).is_err()); + + let invalid_size = skippy_ffi::WorkloadInfoV1 { + struct_size: 0, + ..Default::default() + }; + assert!(WorkloadInfo::try_from(invalid_size).is_err()); + } + #[test] fn loaded_model_flags_classify_state_without_family_names() { assert_eq!( diff --git a/crates/skippy-runtime/src/session.rs b/crates/skippy-runtime/src/session.rs index 56d40b7071..1d3a7c59b8 100644 --- a/crates/skippy-runtime/src/session.rs +++ b/crates/skippy-runtime/src/session.rs @@ -87,6 +87,76 @@ impl StageSession { Ok(()) } + /// Produce one native pooled vector with a positive, caller-verified dimension. + pub fn embed(&mut self, token_ids: &[i32], dimensions: usize) -> Result> { + if dimensions == 0 { + return Err(anyhow!("embedding dimensions must be greater than zero")); + } + let mut output = vec![0.0_f32; dimensions]; + let mut actual_dimensions = 0usize; + let mut error = ptr::null_mut(); + let status = unsafe { + skippy_ffi::skippy_session_embed( + self.raw, + token_ids.as_ptr(), + token_ids.len(), + output.as_mut_ptr(), + output.len(), + &mut actual_dimensions, + &mut error, + ) + }; + ensure_ok(status, error)?; + if actual_dimensions != dimensions { + return Err(anyhow!( + "native embedding dimensions changed from {dimensions} to {actual_dimensions}" + )); + } + self.token_count = u64::try_from(token_ids.len()).context("token count exceeds u64")?; + Ok(output) + } + + /// Score a query/document pair and return the native template's consumed tokens. + pub fn rerank(&mut self, query: &str, document: &str) -> Result<(f32, usize)> { + let query = CString::new(query).context("rerank query contains an interior NUL byte")?; + let document = + CString::new(document).context("rerank document contains an interior NUL byte")?; + let mut score = 0.0_f32; + let mut token_count = 0usize; + let mut error = ptr::null_mut(); + let status = unsafe { + skippy_ffi::skippy_session_rerank( + self.raw, + query.as_ptr(), + document.as_ptr(), + &mut score, + &mut token_count, + &mut error, + ) + }; + ensure_ok(status, error)?; + self.token_count = u64::try_from(token_count).context("token count exceeds u64")?; + Ok((score, token_count)) + } + + /// Encode source tokens and reset decoder position, returning its first input token. + pub fn encode_prompt(&mut self, token_ids: &[i32]) -> Result { + let mut decoder_start_token = 0_i32; + let mut error = ptr::null_mut(); + let status = unsafe { + skippy_ffi::skippy_session_encode_prompt( + self.raw, + token_ids.as_ptr(), + token_ids.len(), + &mut decoder_start_token, + &mut error, + ) + }; + ensure_ok(status, error)?; + self.token_count = 0; + Ok(decoder_start_token) + } + pub fn configure_chat_sampling( &mut self, metadata_json: &str, diff --git a/crates/skippy-server/README.md b/crates/skippy-server/README.md index 76e5972eb5..c103a6fb46 100644 --- a/crates/skippy-server/README.md +++ b/crates/skippy-server/README.md @@ -142,8 +142,9 @@ deadline handling. macOS interface-scoped socket option. In a multi-NIC lab, set `bind_addr` to the private LAN address, such as `192.168.0.x:19031`, so both inbound serving and outbound stage-to-stage traffic are pinned to that interface. -- `serve-openai` exposes `/v1/models`, `/v1/chat/completions`, and - `/v1/completions` using the shared `openai-frontend` crate for a local +- `serve-openai` exposes model discovery, chat/completions, Responses, + embeddings, rerank, and audio endpoints using the shared `openai-frontend` + crate for a local final/single-stage config with no downstream peer. Split serving uses embedded stage-0 OpenAI serving from `serve-binary --openai-bind-addr` because generation-7 prediction returns flow directly from the final stage to stage 0. diff --git a/crates/skippy-server/src/embedded.rs b/crates/skippy-server/src/embedded.rs index 1fee2bfa62..287b7a4a64 100644 --- a/crates/skippy-server/src/embedded.rs +++ b/crates/skippy-server/src/embedded.rs @@ -8,7 +8,7 @@ use anyhow::{Context, Result}; use axum::Router; use openai_frontend::{OpenAiBackend, OpenAiFrontendConfig, OpenAiLifecycleObserver}; use skippy_protocol::{StageConfig, StageTopology}; -use skippy_runtime::{ActivationBoundaryDesc, MtpSource}; +use skippy_runtime::{ActivationBoundaryDesc, MtpSource, WorkloadInfo}; use tokio::{sync::oneshot, task::JoinHandle}; use crate::{ @@ -140,6 +140,23 @@ impl SkippyRuntimeHandle { .output_activation_boundary() } + /// Returns the runtime-probed workload contract for the loaded model. + pub fn workload_info(&self) -> Result { + self.runtime + .lock() + .expect("runtime lock poisoned") + .workload_info() + } + + /// True only when the loaded multimodal projector exposes llama.cpp's + /// audio-generation helper contract. + pub fn supports_speech_synthesis(&self) -> bool { + self.runtime + .lock() + .expect("runtime lock poisoned") + .supports_speech_synthesis() + } + /// Assemble a ready handle around an already-loaded runtime. /// /// Shared by both loaders so the stats cache is primed exactly once, in one diff --git a/crates/skippy-server/src/frontend/backend.rs b/crates/skippy-server/src/frontend/backend.rs index e328b09878..3e3fddbb31 100644 --- a/crates/skippy-server/src/frontend/backend.rs +++ b/crates/skippy-server/src/frontend/backend.rs @@ -16,6 +16,7 @@ use crate::frontend::generation::GenerationSessionLockEntry; use crate::frontend::generation::GenerationStream; use crate::frontend::generation::GenerationStreamEvent; use crate::frontend::generation::GenerationTokenLimit; +use crate::frontend::generation::OpenAiBackendMode; use crate::frontend::generation::OpenAiCacheHints; use crate::frontend::generation::OpenAiGenerationIds; use crate::frontend::generation::PhaseTimer; @@ -35,7 +36,7 @@ use crate::frontend::generation::template_exposes_reasoning; use crate::frontend::request::{ apply_chat_request_defaults, apply_completion_request_defaults, chat_sampling_config, chat_template_options, completion_sampling_config, ensure_chat_runtime_features_supported, - ensure_completion_runtime_features_supported, + ensure_completion_runtime_features_supported, sampling_config, }; use crate::runtime_state::RuntimeSessionStats; use crate::telemetry::Telemetry; @@ -44,6 +45,11 @@ use crate::telemetry::now_unix_nanos; use async_trait::async_trait; use futures_util::StreamExt; use futures_util::stream; +use openai_frontend::AudioFormat; +use openai_frontend::AudioResponse; +use openai_frontend::AudioSpeechRequest; +use openai_frontend::AudioTranscriptionRequest; +use openai_frontend::AudioTranscriptionResponse; use openai_frontend::ChatCompletionOutcome; use openai_frontend::ChatCompletionRequest; use openai_frontend::ChatCompletionResponse; @@ -52,11 +58,18 @@ use openai_frontend::ChatExchangeRoute; use openai_frontend::CompletionRequest; use openai_frontend::CompletionResponse; use openai_frontend::CompletionStream; +use openai_frontend::Embedding; +use openai_frontend::EmbeddingInput; +use openai_frontend::EmbeddingResponse; +use openai_frontend::EmbeddingsRequest; use openai_frontend::ModelObject; use openai_frontend::OpenAiBackend; use openai_frontend::OpenAiError; use openai_frontend::OpenAiRequestContext; use openai_frontend::OpenAiResult; +use openai_frontend::RerankRequest; +use openai_frontend::RerankResponse; +use openai_frontend::RerankResult; use openai_frontend::TerminalGuard; use openai_frontend::TerminalGuardedChatStream; use openai_frontend::apply_chat_hook_outcome; @@ -65,7 +78,10 @@ use openai_frontend::chat_mesh_hooks_enabled; use serde_json::Value; use serde_json::json; use skippy_metrics::attr as attr_key; -use skippy_runtime::SamplingConfig; +use skippy_runtime::{ + MediaInput, ModelWorkload, SamplingConfig, SpeechOutputFormat, SpeechSynthesisConfig, + WorkloadInfo, +}; use std::collections::BTreeMap; use std::sync::Arc; use std::sync::Mutex; @@ -81,6 +97,8 @@ use tokio::sync::mpsc; use tokio::sync::mpsc::error::TrySendError; use tokio::task; +mod non_chat; + fn request_cancelled_error() -> OpenAiError { OpenAiError::cancelled("request cancelled") } @@ -1112,6 +1130,203 @@ impl OpenAiBackend for StageOpenAiBackend { generation_event_to_completion_chunk(event, &model) }))) } + + /// Execute embeddings only through an admitted local full-model workload. + async fn embeddings( + &self, + request: EmbeddingsRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.ensure_model(&request.model)?; + let info = self.ensure_local_workload(ModelWorkload::Embedding)?; + let expected_dimensions = non_chat::embedding_output_dimensions(info.output_dimensions)?; + if request + .dimensions + .is_some_and(|requested| requested != expected_dimensions) + { + return Err(OpenAiError::unsupported(format!( + "model exposes {expected_dimensions} embedding dimensions; dimensionality reduction is not supported" + ))); + } + + let model = request.model.clone(); + let encoding_format = request.encoding_format.clone(); + let backend = self.clone(); + let token_inputs = task::spawn_blocking(move || backend.prepare_embedding_inputs(request)) + .await + .map_err(|error| { + OpenAiError::backend(format!("embedding tokenization task failed: {error}")) + })??; + let prompt_tokens = token_inputs.iter().map(Vec::len).sum::(); + let max_input_tokens = token_inputs.iter().map(Vec::len).max().unwrap_or_default(); + let ids = generation_ids(OpenAiCacheHints::default(), None, &context); + let cancellation = context.cancellation_token(); + let embeddings = self + .run_local_workload( + context, + ids, + max_input_tokens, + move |runtime, session_id| { + non_chat::collect_workload_batch( + token_inputs.iter().enumerate(), + &cancellation, + |(index, tokens)| { + runtime + .embed(session_id, tokens, expected_dimensions) + .map(|values| Embedding { values, index }) + }, + ) + }, + ) + .await?; + Ok(EmbeddingResponse::from_embeddings( + model, + embeddings, + u32::try_from(prompt_tokens).unwrap_or(u32::MAX), + &encoding_format, + )) + } + + /// Execute reranking only through an admitted local full-model workload. + async fn rerank( + &self, + request: RerankRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.ensure_model(&request.model)?; + self.ensure_local_workload(ModelWorkload::Rerank)?; + let prompt_tokens_estimate = non_chat::rerank_prompt_tokens_estimate(&request)?; + let ids = generation_ids(OpenAiCacheHints::default(), None, &context); + let cancellation = context.cancellation_token(); + let query = request.query.clone(); + let documents = request.documents.clone(); + let mut scored = self + .run_local_workload( + context, + ids, + prompt_tokens_estimate, + move |runtime, session_id| { + non_chat::collect_workload_batch( + documents.iter().enumerate(), + &cancellation, + |(index, document)| { + let (relevance_score, token_count) = + runtime.rerank(session_id, &query, document.text()?)?; + Ok((index, relevance_score, token_count)) + }, + ) + }, + ) + .await?; + let prompt_tokens = scored.iter().map(|(_, _, count)| count).sum::(); + scored.sort_by(|left, right| right.1.total_cmp(&left.1).then(left.0.cmp(&right.0))); + scored.truncate(request.top_n.unwrap_or(scored.len()).min(scored.len())); + let results = scored + .into_iter() + .map(|(index, relevance_score, _)| RerankResult { + index, + relevance_score, + document: request + .return_documents + .then(|| request.documents[index].clone()), + }) + .collect(); + Ok(RerankResponse { + id: format!("rerank-{}", uuid::Uuid::new_v4().simple()), + results, + usage: openai_frontend::Usage { + prompt_tokens: u32::try_from(prompt_tokens).unwrap_or(u32::MAX), + completion_tokens: 0, + total_tokens: u32::try_from(prompt_tokens).unwrap_or(u32::MAX), + ..openai_frontend::Usage::default() + }, + }) + } + + /// Execute native speech synthesis with the configured local projector. + async fn audio_speech( + &self, + request: AudioSpeechRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.ensure_model(&request.model)?; + if !self.has_unsplit_full_model_topology() { + return Err(OpenAiError::unsupported( + "speech synthesis currently requires an unsplit local runtime", + )); + } + if (request.speed - 1.0).abs() > f32::EPSILON { + return Err(OpenAiError::unsupported( + "speech speed control is not supported by the native model", + )); + } + non_chat::validate_speech_voice(&request.voice)?; + let output_format = match request.response_format { + AudioFormat::Wav => SpeechOutputFormat::Wav, + AudioFormat::Pcm => SpeechOutputFormat::PcmS16Le, + _ => { + return Err(OpenAiError::unsupported( + "native speech synthesis currently supports wav and pcm output", + )); + } + }; + { + let runtime = self + .runtime + .lock() + .map_err(|_| OpenAiError::backend("runtime lock poisoned"))?; + if runtime.input_activation_boundary().is_some() + || runtime.output_activation_boundary().is_some() + || !runtime.supports_speech_synthesis() + { + return Err(OpenAiError::unsupported( + "model does not expose full-model speech synthesis", + )); + } + } + let content_type = request.response_format.content_type().to_string(); + let prompt_tokens_estimate = request.input.len().div_ceil(3).max(1); + let ids = generation_ids(OpenAiCacheHints::default(), None, &context); + let cancellation = context.clone(); + let config = SpeechSynthesisConfig { + prompt: request.input, + language: None, + top_k: 20, + top_p: 0.8, + seed: u32::MAX, + output_format, + max_frames: 512, + }; + let audio = self + .run_local_workload( + context, + ids, + prompt_tokens_estimate, + move |runtime, session_id| { + runtime.synthesize_speech(session_id, &config, || cancellation.is_cancelled()) + }, + ) + .await?; + AudioResponse::new(audio.bytes, content_type) + } + + /// Decode uploaded audio using the local speech-recognition runtime. + async fn audio_transcription( + &self, + request: AudioTranscriptionRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.audio_to_text(request, false, context).await + } + + /// Translate uploaded audio using the local speech-recognition runtime. + async fn audio_translation( + &self, + request: AudioTranscriptionRequest, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.audio_to_text(request, true, context).await + } } impl StageOpenAiBackend { diff --git a/crates/skippy-server/src/frontend/backend/non_chat.rs b/crates/skippy-server/src/frontend/backend/non_chat.rs new file mode 100644 index 0000000000..9d0f0aaa91 --- /dev/null +++ b/crates/skippy-server/src/frontend/backend/non_chat.rs @@ -0,0 +1,508 @@ +use super::*; + +/// Reject an invalid model descriptor before allocating any embedding sessions. +pub(super) fn embedding_output_dimensions(dimensions: u32) -> OpenAiResult { + if dimensions == 0 { + return Err(OpenAiError::backend( + "model did not report an embedding output dimension", + )); + } + usize::try_from(dimensions) + .map_err(|_| OpenAiError::backend("embedding dimensions exceed usize")) +} + +/// Validate the same documents that execution consumes before estimating admission. +pub(super) fn rerank_prompt_tokens_estimate(request: &RerankRequest) -> OpenAiResult { + request.validate()?; + request + .documents + .iter() + .map(|document| { + document + .text() + .map(|text| request.query.len().saturating_add(text.len()).div_ceil(3)) + }) + .try_fold(1, |estimate, next| next.map(|tokens| estimate.max(tokens))) +} + +/// Stop a native batch between items when its owning HTTP request is cancelled. +pub(super) fn collect_workload_batch( + items: I, + cancellation: &openai_frontend::CancellationToken, + mut run_item: F, +) -> anyhow::Result> +where + I: IntoIterator, + F: FnMut(I::Item) -> anyhow::Result, +{ + let mut results = Vec::new(); + for item in items { + // Native calls cannot be interrupted mid-item; stop before the next + // one so a disconnected client does not hold the runtime for a batch. + if cancellation.is_cancelled() { + return Err(request_cancelled_error().into()); + } + results.push(run_item(item)?); + } + Ok(results) +} + +/// Preserve structured frontend errors while adding context to native failures. +fn workload_error(error: anyhow::Error) -> OpenAiError { + if let Some(openai_error) = error.downcast_ref::() { + return openai_error.clone(); + } + OpenAiError::backend(format!("workload execution failed: {error:#}")) +} + +/// Accept only the implemented default speaker; do not reinterpret voice as language. +pub(super) fn validate_speech_voice(voice: &str) -> OpenAiResult<()> { + if voice == "default" { + Ok(()) + } else { + Err(OpenAiError::unsupported( + "native speech synthesis currently supports only voice 'default'; speaker selection is not implemented", + ) + .with_param("voice")) + } +} + +impl StageOpenAiBackend { + /// Recognize complete local models in both standalone and embedded serving modes. + pub(in crate::frontend) fn has_unsplit_full_model_topology(&self) -> bool { + /// Accept full-model execution only when the topology has no remote stage boundaries. + fn is_unsplit(config: &skippy_protocol::StageConfig) -> bool { + config.stage_index == 0 + && config.layer_start == 0 + && config.layer_end > 0 + && !config.filter_tensors_on_load + && config.upstream.is_none() + && config.downstream.is_none() + } + + // The mesh serves a complete local GGUF through EmbeddedStageZero as + // well as through LocalRuntime. The mode name alone cannot tell us + // whether model execution is distributed. + is_unsplit(&self.config) + && match &self.mode { + OpenAiBackendMode::LocalRuntime => true, + OpenAiBackendMode::EmbeddedStageZero { config, .. } => is_unsplit(config), + } + } + + /// Run audio transcription or English translation through a full-model projector. + pub(super) async fn audio_to_text( + &self, + request: AudioTranscriptionRequest, + translate_to_english: bool, + context: OpenAiRequestContext, + ) -> OpenAiResult { + self.ensure_model(&request.model)?; + if !self.has_unsplit_full_model_topology() { + return Err(OpenAiError::unsupported( + "audio transcription currently requires an unsplit local runtime", + )); + } + { + let runtime = self + .runtime + .lock() + .map_err(|_| OpenAiError::backend("runtime lock poisoned"))?; + if runtime.input_activation_boundary().is_some() + || runtime.output_activation_boundary().is_some() + || !runtime.has_media_projector() + { + return Err(OpenAiError::unsupported( + "model does not expose full-model multimodal audio input", + )); + } + } + + let instruction = audio_text_instruction(&request, translate_to_english); + let mut chat_request = ChatCompletionRequest { + model: request.model.clone(), + messages: vec![audio_text_user_message(instruction)], + temperature: request.temperature, + ..Default::default() + }; + apply_chat_request_defaults(&mut chat_request, &self.request_defaults)?; + let template_options = chat_template_options(&chat_request, &self.request_defaults)?; + let mut prompt = self + .prepare_chat_prompt_offloaded(&chat_request, template_options) + .await?; + prompt.media = vec![MediaInput { + bytes: request.file, + }]; + let sampling = sampling_config( + chat_request.temperature, + chat_request.top_p, + chat_request.presence_penalty, + chat_request.frequency_penalty, + chat_request.seed, + chat_request.logit_bias.as_ref(), + &chat_request.extra, + )?; + let ids = generation_ids(OpenAiCacheHints::default(), None, &context); + let output = self + .run_generation( + prompt, + GenerationTokenLimit::from_request(None, self.default_max_tokens), + None, + sampling, + None, + context, + ids, + ) + .await?; + Ok(AudioTranscriptionResponse { + text: audio_transcript_text(&output.text), + }) + } + + /// Reject split execution and mismatched native workload descriptors before work. + pub(in crate::frontend) fn ensure_local_workload( + &self, + expected: ModelWorkload, + ) -> OpenAiResult { + if !self.has_unsplit_full_model_topology() { + return Err(OpenAiError::unsupported( + "non-chat workloads currently require an unsplit local runtime", + )); + } + let runtime = self + .runtime + .lock() + .map_err(|_| OpenAiError::backend("runtime lock poisoned"))?; + if runtime.input_activation_boundary().is_some() + || runtime.output_activation_boundary().is_some() + { + return Err(OpenAiError::unsupported( + "non-chat workloads currently require an unsplit full model", + )); + } + let info = runtime + .workload_info() + .map_err(|error| OpenAiError::backend(format!("read model workload: {error:#}")))?; + if info.kind != expected { + return Err(OpenAiError::unsupported(format!( + "model workload is {:?}; endpoint requires {:?}", + info.kind, expected + ))); + } + Ok(info) + } + + /// Tokenize text batches with the loaded vocabulary, preserving supplied token IDs. + pub(super) fn prepare_embedding_inputs( + &self, + request: EmbeddingsRequest, + ) -> OpenAiResult>> { + let reader = self + .runtime + .lock() + .map_err(|_| OpenAiError::backend("runtime lock poisoned"))? + .model + .reader(); + match request.input { + EmbeddingInput::Text(text) => reader + .tokenize(&text, true) + .map(|tokens| vec![tokens]) + .map_err(|error| OpenAiError::backend(format!("tokenize embedding: {error:#}"))), + EmbeddingInput::Texts(texts) => texts + .into_iter() + .map(|text| { + reader.tokenize(&text, true).map_err(|error| { + OpenAiError::backend(format!("tokenize embedding: {error:#}")) + }) + }) + .collect(), + EmbeddingInput::Tokens(tokens) => Ok(vec![tokens]), + EmbeddingInput::TokenArrays(tokens) => Ok(tokens), + } + } + + /// Execute admitted blocking work with cancellation, slot ownership, and cleanup. + pub(super) async fn run_local_workload( + &self, + context: OpenAiRequestContext, + ids: OpenAiGenerationIds, + prompt_tokens: usize, + work: F, + ) -> OpenAiResult + where + T: Send + 'static, + F: FnOnce(&mut crate::runtime_state::RuntimeState, &str) -> anyhow::Result + + Send + + 'static, + { + let cancellation = context.cancellation_token(); + let (permit, session_permit) = self + .acquire_generation_admission( + &ids, + &cancellation, + GenerationAdmissionWork::new(prompt_tokens.max(1), 0), + GenerationAdmissionScheduling::default(), + ) + .await?; + let runtime = Arc::clone(&self.runtime); + let session_id = ids.session_id_string(); + let worker_context = context.clone(); + let result = run_blocking_generation_worker(permit, context, move |token| { + let _session_permit = session_permit; + if token.is_cancelled() { + return Err(request_cancelled_error()); + } + let mut runtime = runtime + .lock() + .map_err(|_| OpenAiError::backend("runtime lock poisoned"))?; + let result = work(&mut runtime, &session_id).map_err(workload_error); + let cleanup = runtime.drop_session_timed(&session_id).map_err(|error| { + OpenAiError::backend(format!("workload session cleanup failed: {error:#}")) + }); + match (result, cleanup) { + (Ok(value), Ok(_)) => Ok(value), + (Err(error), _) => Err(error), + (Ok(_), Err(error)) => Err(error), + } + }) + .await + .map_err(|error| OpenAiError::backend(format!("workload task failed: {error}")))?; + if worker_context.is_cancelled() { + Err(request_cancelled_error()) + } else { + result + } + } +} + +/// Remove only recognized model-generated transcript wrappers. +fn audio_transcript_text(raw: &str) -> String { + let text = raw.trim(); + for prefix in ["The text is:", "The audio is:"] { + if let Some(quoted) = text.strip_prefix(prefix).map(str::trim) + && let Some(inner) = quoted + .strip_prefix('"') + .and_then(|value| value.strip_suffix('"')) + { + return inner.to_string(); + } + } + text.to_string() +} + +/// Choose the transcription or translation instruction without changing supplied context. +fn audio_text_instruction( + request: &AudioTranscriptionRequest, + translate_to_english: bool, +) -> String { + let mut instruction = if translate_to_english { + "Translate the supplied audio into English. Return only the English translation." + .to_string() + } else { + let language = request + .language + .as_deref() + .filter(|language| !language.trim().is_empty()) + .map_or_else(String::new, |language| format!(" (language: {language})")); + format!("Transcribe audio to text{language}") + }; + if let Some(prompt) = request + .prompt + .as_deref() + .filter(|prompt| !prompt.trim().is_empty()) + { + instruction.push_str(" Use this context when resolving names or terminology: "); + instruction.push_str(prompt); + } + instruction +} + +/// Place the audio marker after the task instruction and optional prompt. +fn audio_text_user_message(instruction: String) -> openai_frontend::ChatMessage { + openai_frontend::ChatMessage { + role: "user".to_string(), + // The audio marker must follow the instruction. Upstream llama.cpp's + // transcription route appends the media marker to the user prompt; + // reversing these parts can make Ultravox ignore the supplied audio. + content: Some(openai_frontend::MessageContent::Parts(vec![ + openai_frontend::MessageContentPart { + content_type: "text".to_string(), + text: Some(instruction), + extra: BTreeMap::new(), + }, + openai_frontend::MessageContentPart { + content_type: "input_audio".to_string(), + text: None, + extra: BTreeMap::from([( + "input_audio".to_string(), + json!({"data": "AA==", "format": "wav"}), + )]), + }, + ])), + extra: BTreeMap::new(), + } +} + +#[cfg(test)] +mod tests { + #[test] + /// Reject zero output width before reserving a native embedding session. + fn invalid_embedding_dimensions_are_rejected_at_admission() { + assert!( + super::embedding_output_dimensions(0) + .unwrap_err() + .body() + .error + .message + .contains("did not report") + ); + assert_eq!(super::embedding_output_dimensions(768).unwrap(), 768); + } + + #[test] + /// Cost estimation must validate every document, including entries after valid ones. + fn rerank_estimate_rejects_invalid_documents_before_workload_admission() { + let mut request: openai_frontend::RerankRequest = + serde_json::from_value(serde_json::json!({ + "model": "rank", "query": "query", "documents": ["text", {"title": "no text"}] + })) + .unwrap(); + assert!(super::rerank_prompt_tokens_estimate(&request).is_err()); + request.documents.pop(); + assert_eq!(super::rerank_prompt_tokens_estimate(&request).unwrap(), 3); + } + + use super::{ + audio_text_instruction, audio_text_user_message, audio_transcript_text, + collect_workload_batch, validate_speech_voice, workload_error, + }; + use openai_frontend::{AudioTranscriptionRequest, MessageContent}; + + /// Construct a bounded transcription request for prompt and normalization tests. + fn audio_request() -> AudioTranscriptionRequest { + AudioTranscriptionRequest { + model: "ultravox".to_string(), + file: vec![1], + filename: Some("sample.wav".to_string()), + language: None, + prompt: None, + response_format: "json".to_string(), + temperature: Some(0.0), + } + } + + #[test] + /// Default transcription uses the reference prompt with its audio marker last. + fn transcription_uses_upstream_default_prompt_before_audio_marker() { + let instruction = audio_text_instruction(&audio_request(), false); + assert_eq!(instruction, "Transcribe audio to text"); + + let message = audio_text_user_message(instruction); + let Some(MessageContent::Parts(parts)) = message.content else { + panic!("audio request must use multipart chat content"); + }; + assert_eq!(parts.len(), 2); + assert_eq!(parts[0].content_type, "text"); + assert_eq!(parts[0].text.as_deref(), Some("Transcribe audio to text")); + assert_eq!(parts[1].content_type, "input_audio"); + assert!(parts[1].extra.contains_key("input_audio")); + } + + #[test] + /// Normalize confirmed decoration without deleting ordinary transcript content. + fn transcription_removes_only_confirmed_quoted_ultravox_wrappers() { + assert_eq!( + audio_transcript_text("The text is: \"The mesh is ready\""), + "The mesh is ready" + ); + assert_eq!( + audio_transcript_text("The audio is: \"The mesh is ready\""), + "The mesh is ready" + ); + assert_eq!( + audio_transcript_text(" The mesh is ready "), + "The mesh is ready" + ); + assert_eq!( + audio_transcript_text("The text is: The mesh is ready"), + "The text is: The mesh is ready" + ); + assert_eq!( + audio_transcript_text("I heard: \"The mesh is ready\""), + "I heard: \"The mesh is ready\"" + ); + } + + #[test] + /// Empty native output remains empty rather than becoming fabricated transcript text. + fn transcription_preserves_empty_text_without_an_artificial_wrapper() { + assert_eq!(audio_transcript_text(" "), ""); + assert_eq!(audio_transcript_text("The text is: \"\""), ""); + assert_eq!(audio_transcript_text("The audio is: \"\""), ""); + } + + #[test] + /// Task selection and user context must not move the media marker ahead of instructions. + fn translation_and_explicit_context_keep_audio_marker_last() { + let mut request = audio_request(); + request.language = Some("German".to_string()); + request.prompt = Some("mesh-llm".to_string()); + assert_eq!( + audio_text_instruction(&request, false), + "Transcribe audio to text (language: German) Use this context when resolving names or terminology: mesh-llm" + ); + + let instruction = audio_text_instruction(&request, true); + assert!(instruction.starts_with("Translate the supplied audio into English.")); + let message = audio_text_user_message(instruction); + let Some(MessageContent::Parts(parts)) = message.content else { + panic!("audio translation must use multipart chat content"); + }; + assert_eq!(parts[0].content_type, "text"); + assert_eq!(parts[1].content_type, "input_audio"); + } + + #[test] + /// Voice selection must not be silently repurposed as a language selector. + fn speech_voice_rejects_unsupported_speakers_without_reinterpreting_language() { + validate_speech_voice("default").expect("the default native speaker is supported"); + + for voice in ["alloy", "english", ""] { + let error = validate_speech_voice(voice).expect_err("speaker selection is unsupported"); + assert_eq!( + error.body().error.code.as_deref(), + Some("unsupported_model_feature") + ); + assert_eq!(error.body().error.param.as_deref(), Some("voice")); + assert!(error.body().error.message.contains("voice 'default'")); + } + } + + #[test] + /// Cancellation between batch items prevents any further native execution. + fn workload_batch_stops_before_the_next_native_call_after_cancellation() { + let cancellation = openai_frontend::CancellationToken::new(); + let mut calls = 0; + let result = collect_workload_batch(0..3, &cancellation, |item| { + calls += 1; + cancellation.cancel(); + Ok(item) + }); + let error = result.expect_err("the remaining batch must stop after cancellation"); + assert_eq!(calls, 1); + assert_eq!( + workload_error(error).body().error.code.as_deref(), + Some("request_cancelled") + ); + } + + #[test] + /// Preserve cancellation classification through workload error translation. + fn workload_error_keeps_structured_cancellation() { + let error = workload_error(anyhow::Error::new(super::request_cancelled_error())); + assert_eq!( + error.body().error.code.as_deref(), + Some("request_cancelled") + ); + } +} diff --git a/crates/skippy-server/src/frontend/backend/tests.rs b/crates/skippy-server/src/frontend/backend/tests.rs index c9ab775f71..5d75310ff4 100644 --- a/crates/skippy-server/src/frontend/backend/tests.rs +++ b/crates/skippy-server/src/frontend/backend/tests.rs @@ -5,6 +5,7 @@ use crate::frontend::admission::GenerationTokenBudget; use crate::frontend::generation::ADMISSION_STARVATION_BOUND_TURNS; use crate::frontend::generation::OpenAiBackendMode; use crate::frontend::iteration_scheduler::IterationScheduler; +use crate::frontend::prefill::PrefillChunkPolicy; use crate::runtime_state::RuntimeState; use futures_util::StreamExt; use openai_frontend::ChatCompletionChunk; @@ -1248,6 +1249,78 @@ fn hooks_test_backend(hook_policy: Option>) -> StageOp } } +/// Construct a stage-zero embedded topology for non-chat admission tests. +fn embedded_non_chat_test_mode(config: skippy_protocol::StageConfig) -> OpenAiBackendMode { + OpenAiBackendMode::EmbeddedStageZero { + config, + prefill_chunk_policy: PrefillChunkPolicy::Fixed { chunk_size: 64 }, + activation_width: 0, + downstream_wire_condition: crate::binary_transport::WireCondition::new(0.0, None) + .expect("unconditioned test wire"), + prefill_reply_credit_limit: 0, + lane_pool: None, + prediction_returns: None, + } +} + +#[test] +/// Allow non-chat execution for a complete, unsplit embedded model. +fn embedded_stage_zero_admits_unsplit_local_non_chat_topology() { + let mut backend = hooks_test_backend(None); + backend.mode = embedded_non_chat_test_mode(backend.config.clone()); + + assert!(backend.has_unsplit_full_model_topology()); +} + +#[test] +/// Reject non-chat execution with incomplete layer, tensor, or expert ownership. +fn non_chat_topology_guard_rejects_staged_and_filtered_models() { + let mut backend = hooks_test_backend(None); + let full_config = backend.config.clone(); + let mut cases = Vec::new(); + + let mut downstream = full_config.clone(); + downstream.downstream = Some(skippy_protocol::PeerConfig { + stage_id: "stage-1".to_string(), + stage_index: 1, + endpoint: "127.0.0.1:0".to_string(), + }); + cases.push(downstream); + + let mut upstream = full_config.clone(); + upstream.upstream = Some(skippy_protocol::PeerConfig { + stage_id: "stage-previous".to_string(), + stage_index: 0, + endpoint: "127.0.0.1:0".to_string(), + }); + cases.push(upstream); + + let mut filtered = full_config.clone(); + filtered.filter_tensors_on_load = true; + cases.push(filtered); + + let mut partial = full_config.clone(); + partial.layer_start = 1; + cases.push(partial); + + let mut non_first_stage = full_config.clone(); + non_first_stage.stage_index = 1; + cases.push(non_first_stage); + + let mut empty = full_config.clone(); + empty.layer_end = 0; + cases.push(empty); + + for config in cases { + backend.mode = embedded_non_chat_test_mode(config.clone()); + assert!(!backend.has_unsplit_full_model_topology(), "{config:?}"); + backend.mode = OpenAiBackendMode::LocalRuntime; + backend.config = config; + assert!(!backend.has_unsplit_full_model_topology()); + backend.config = full_config.clone(); + } +} + fn mesh_hooks_request(model: &str) -> ChatCompletionRequest { let mut request: ChatCompletionRequest = serde_json::from_value(json!({ "model": model, diff --git a/crates/skippy-server/src/frontend/generation_flow.rs b/crates/skippy-server/src/frontend/generation_flow.rs index cb9e9d5972..e5e3752801 100644 --- a/crates/skippy-server/src/frontend/generation_flow.rs +++ b/crates/skippy-server/src/frontend/generation_flow.rs @@ -1,3 +1,4 @@ +mod encoder_decoder; mod text_generation; use crate::binary_transport::forwarded_stage_message_timed; diff --git a/crates/skippy-server/src/frontend/generation_flow/encoder_decoder.rs b/crates/skippy-server/src/frontend/generation_flow/encoder_decoder.rs new file mode 100644 index 0000000000..9946e80cbc --- /dev/null +++ b/crates/skippy-server/src/frontend/generation_flow/encoder_decoder.rs @@ -0,0 +1,96 @@ +use std::time::Instant; + +use openai_frontend::{ChatCompletionRequest, OpenAiError, OpenAiResult}; +use skippy_runtime::{ModelWorkload, SamplingConfig}; + +use crate::frontend::generation::{ + GenerationCacheStats, OpenAiGenerationIds, StageOpenAiBackend, TokenControl, + tool_calls_requested, +}; +use crate::frontend::util::openai_backend_error; + +use super::LocalSessionCleanupGuard; + +impl StageOpenAiBackend { + #[allow(clippy::too_many_arguments)] + /// Encode once, decode bounded tokens, and clean up the exclusive session on every exit. + pub(super) fn generate_encoder_decoder_tokens( + &self, + prompt_token_ids: &[i32], + max_tokens: u32, + sampling: &SamplingConfig, + chat_request: Option<&ChatCompletionRequest>, + cancellation: Option<&openai_frontend::CancellationToken>, + ids: &OpenAiGenerationIds, + mut emit_token: impl FnMut(i32) -> OpenAiResult, + ) -> OpenAiResult { + if !self.has_unsplit_full_model_topology() { + return Err(OpenAiError::unsupported( + "encoder-decoder models currently require an unsplit local runtime", + )); + } + self.ensure_local_workload(ModelWorkload::EncoderDecoder)?; + if chat_request.is_some_and(tool_calls_requested) { + return Err(OpenAiError::unsupported( + "tool calls are not supported by encoder-decoder models", + )); + } + let session_id = ids.session_label.clone(); + let (result, mut cleanup) = LocalSessionCleanupGuard::run( + || self.cleanup_local_generation_session(&session_id, ids), + || { + let mut runtime = self + .runtime + .lock() + .map_err(|_| OpenAiError::backend("runtime lock poisoned"))?; + let prompt_started = Instant::now(); + let decoder_start = runtime + .encode_prompt(&session_id, prompt_token_ids) + .map_err(openai_backend_error)?; + let prompt_ms = prompt_started.elapsed().as_secs_f64() * 1_000.0; + let predicted_started = Instant::now(); + if max_tokens > 0 { + let mut predicted = runtime + .decode_frame_sampled( + &session_id, + decoder_start, + sampling.enabled.then_some(sampling), + None, + 0, + ) + .map_err(openai_backend_error)? + .0; + for generated in 0..max_tokens { + if cancellation + .is_some_and(openai_frontend::CancellationToken::is_cancelled) + { + return Err(OpenAiError::backend("request cancelled")); + } + if emit_token(predicted)? == TokenControl::Stop { + break; + } + if generated + 1 < max_tokens { + predicted = runtime + .decode_frame_sampled( + &session_id, + predicted, + sampling.enabled.then_some(sampling), + None, + 0, + ) + .map_err(openai_backend_error)? + .0; + } + } + } + Ok(GenerationCacheStats { + prompt_ms, + predicted_ms: predicted_started.elapsed().as_secs_f64() * 1_000.0, + ..GenerationCacheStats::default() + }) + }, + ); + cleanup.cleanup(); + result + } +} diff --git a/crates/skippy-server/src/frontend/generation_flow/text_generation.rs b/crates/skippy-server/src/frontend/generation_flow/text_generation.rs index 02d28d694d..e347f6f322 100644 --- a/crates/skippy-server/src/frontend/generation_flow/text_generation.rs +++ b/crates/skippy-server/src/frontend/generation_flow/text_generation.rs @@ -6,7 +6,7 @@ use crate::frontend::generation::{ use crate::frontend::util::{generation_stop_values, openai_backend_error}; use openai_frontend::{ChatCompletionRequest, OpenAiError, OpenAiResult}; use serde_json::json; -use skippy_runtime::SamplingConfig; +use skippy_runtime::{ModelWorkload, SamplingConfig}; pub(super) fn resident_capacity_target_tokens(prompt_token_count: usize) -> u64 { u64::try_from(prompt_token_count).unwrap_or(u64::MAX) @@ -91,6 +91,33 @@ impl StageOpenAiBackend { Some(prepared) => prepared, None => self.prepare_text_prompt(&prompt, max_tokens, &ids)?, }; + let workload = self + .runtime + .lock() + .map_err(|_| OpenAiError::backend("runtime lock poisoned"))? + .workload_info() + .map_err(openai_backend_error)? + .kind; + if workload == ModelWorkload::EncoderDecoder { + let mut collector = + TextGenerationCollector::new(self.runtime.clone(), stop_values, on_text_chunk)? + .with_ignore_eos(sampling.ignore_eos); + let cache_stats = self.generate_encoder_decoder_tokens( + &prompt_token_ids, + max_tokens, + &sampling, + hook_request.as_ref(), + cancellation, + &ids, + |token| collector.push_token(token), + )?; + return collector.finish(prompt_token_ids.len(), cache_stats); + } + if workload != ModelWorkload::CausalGeneration { + return Err(OpenAiError::unsupported(format!( + "model workload is {workload:?}; chat and completion endpoints require a generative model" + ))); + } // This is an optional cache candidate. The already-rendered prompt is // valid even when its second, assistant-marker-free rendering cannot // be tokenized, so bypass the candidate rather than failing the chat diff --git a/crates/skippy-server/src/frontend/tests/mod.rs b/crates/skippy-server/src/frontend/tests/mod.rs index 11b3e48f89..9cb532db2d 100644 --- a/crates/skippy-server/src/frontend/tests/mod.rs +++ b/crates/skippy-server/src/frontend/tests/mod.rs @@ -13,7 +13,7 @@ pub(super) use crate::kv_integration::PrefillKvIdentity; pub(super) use crate::kv_integration::{ KvLifecycleEvent, KvLifecycleObserver, KvStageIntegration, proactive_eviction_attrs, }; -pub(super) use crate::runtime_state::load_runtime; +pub(super) use crate::runtime_state::{load_runtime, reject_unsupported_staged_workload}; pub(super) use crate::telemetry::Telemetry; pub(super) use anyhow::{Context as _, Result, anyhow, bail}; pub(super) use async_trait::async_trait; @@ -29,8 +29,8 @@ pub(super) use openai_frontend::{ pub(super) use serde_json::{Value, json}; pub(super) use skippy_metrics::attr as attr_key; pub(super) use skippy_protocol::{ - LoadMode, MessageBase, PeerConfig, SCHEMA_VERSION, StageConfig, StageKvCacheConfig, - StageKvCacheMode, StageKvCachePayload, + LoadMode, MessageBase, PeerConfig, SCHEMA_VERSION, StageConfig, StageDevice, + StageKvCacheConfig, StageKvCacheMode, StageKvCachePayload, binary::{LLAMA_TOKEN_NULL, StageReplyStats, WireMessageKind, write_stage_message}, }; pub(super) use skippy_runtime::{ @@ -54,10 +54,12 @@ mod draft_runtime; mod generation; mod guardrails; mod multimodal; +mod non_chat; mod prefill; mod prefix_cache; mod prefix_cache_lifecycle; mod prompting; mod request; mod support; +mod tts_oracle; mod wire_messages; diff --git a/crates/skippy-server/src/frontend/tests/multimodal.rs b/crates/skippy-server/src/frontend/tests/multimodal.rs index 48e2426c0c..b0c264e496 100644 --- a/crates/skippy-server/src/frontend/tests/multimodal.rs +++ b/crates/skippy-server/src/frontend/tests/multimodal.rs @@ -223,47 +223,6 @@ fn multimodal_stage_config( } } -fn local_openai_backend(config: StageConfig) -> Result { - let runtime = load_runtime(&config)?.context("load smoke runtime")?; - let ctx_size = usize::try_from(config.ctx_size).unwrap_or(usize::MAX); - let telemetry = Telemetry::new( - None, - 1, - config.clone(), - crate::telemetry::TelemetryLevel::Off, - ); - let iteration_scheduler = - IterationScheduler::new(runtime.clone(), &config, 1, true, telemetry.clone())?; - Ok(StageOpenAiBackend { - runtime, - telemetry, - config, - model_id: "mm-smoke".to_string(), - default_max_tokens: 16, - request_defaults: EmbeddedOpenAiRequestDefaults::default(), - ctx_size, - mode: OpenAiBackendMode::LocalRuntime, - draft: None, - speculative_window: 0, - adaptive_speculative_window: false, - ngram_max: 0, - speculative: SpeculativeDecodeConfig::default(), - generation_limit: Arc::new(GenerationConcurrencyController::fixed(1)), - generation_queue_depth: Arc::new(AtomicUsize::new(0)), - generation_queue_limit: 1, - generation_admission_timeout: std::time::Duration::from_secs(10), - generation_service_estimator: Arc::new(crate::frontend::GenerationServiceEstimator::new(1)), - generation_session_locks: Arc::new(Mutex::new(std::collections::BTreeMap::new())), - generation_token_budget: Arc::new(GenerationTokenBudget::new(ctx_size)), - hook_policy: None, - generation_receipt: None, - generation_lifecycle: None, - linear_proposal_ingress: None, - kv: None, - iteration_scheduler, - }) -} - fn multimodal_chat_request(fixture: &MultimodalSmokeFixture) -> Result { multimodal_chat_request_with_max_tokens(fixture, fixture.max_tokens) } @@ -371,7 +330,7 @@ async fn real_multimodal_local_smoke_when_fixture_is_set() -> Result<()> { fixture.layer_end, available_loopback_addr()?, ); - let backend = local_openai_backend(config)?; + let backend = support::local_openai_backend(config, "mm-smoke")?; let response = backend .chat_completion(multimodal_chat_request(&fixture)?) .await?; @@ -393,7 +352,7 @@ async fn real_multimodal_local_prefill_failure_releases_lane_when_fixture_is_set fixture.layer_end, available_loopback_addr()?, ); - let backend = local_openai_backend(config)?; + let backend = support::local_openai_backend(config, "mm-smoke")?; for attempt in 0..2 { let result = backend @@ -436,7 +395,7 @@ async fn real_multimodal_length_limit_then_next_image_uses_clean_lane_when_fixtu available_loopback_addr()?, ); config.lane_count = 2; - let backend = local_openai_backend(config)?; + let backend = support::local_openai_backend(config, "mm-smoke")?; backend .runtime .lock() diff --git a/crates/skippy-server/src/frontend/tests/non_chat.rs b/crates/skippy-server/src/frontend/tests/non_chat.rs new file mode 100644 index 0000000000..b5126e000d --- /dev/null +++ b/crates/skippy-server/src/frontend/tests/non_chat.rs @@ -0,0 +1,517 @@ +use super::*; + +use openai_frontend::{ + AudioFormat, AudioSpeechRequest, AudioTranscriptionRequest, EmbeddingInput, EmbeddingOutput, + EmbeddingsRequest, RerankDocument, RerankRequest, RerankResult, +}; +use skippy_runtime::ModelWorkload; + +const MODEL_ENV: &str = "SKIPPY_WORKLOAD_MODEL"; +const MODEL_ID_ENV: &str = "SKIPPY_WORKLOAD_MODEL_ID"; +const CLASS_ENV: &str = "SKIPPY_WORKLOAD_CLASS"; +const PROJECTOR_ENV: &str = "SKIPPY_WORKLOAD_PROJECTOR"; +const MEDIA_ENV: &str = "SKIPPY_WORKLOAD_MEDIA"; +const LAYER_END_ENV: &str = "SKIPPY_WORKLOAD_LAYER_END"; +const CTX_SIZE_ENV: &str = "SKIPPY_WORKLOAD_CTX_SIZE"; +const MAX_TOKENS_ENV: &str = "SKIPPY_WORKLOAD_MAX_TOKENS"; +const N_GPU_LAYERS_ENV: &str = "SKIPPY_WORKLOAD_N_GPU_LAYERS"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum CertifiedWorkloadClass { + Embedding, + Rerank, + EncoderDecoder, + Ocr, + SpeechSynthesis, + SpeechRecognition, +} + +impl CertifiedWorkloadClass { + /// Decode the closed workload-class vocabulary used by certification fixtures. + fn parse(value: &str) -> Result { + match value { + "embedding" => Ok(Self::Embedding), + "rerank" => Ok(Self::Rerank), + "encoder_decoder" => Ok(Self::EncoderDecoder), + "ocr" => Ok(Self::Ocr), + "speech_synthesis" => Ok(Self::SpeechSynthesis), + "speech_recognition" => Ok(Self::SpeechRecognition), + other => bail!("unsupported {CLASS_ENV} value {other:?}"), + } + } + + /// Identify classes whose native execution requires a pinned projector sidecar. + fn requires_projector(self) -> bool { + matches!( + self, + Self::Ocr | Self::SpeechSynthesis | Self::SpeechRecognition + ) + } + + /// Identify classes whose smoke needs a deterministic media input. + fn requires_media(self) -> bool { + matches!(self, Self::Ocr | Self::SpeechRecognition) + } + + /// Return the class label expected in fail-closed staged-execution errors. + fn staging_label(self) -> Option<&'static str> { + match self { + Self::Embedding => Some("embedding"), + Self::Rerank => Some("rerank"), + Self::EncoderDecoder => Some("encoder_decoder"), + Self::SpeechSynthesis => Some("speech_synthesis"), + Self::Ocr | Self::SpeechRecognition => None, + } + } +} + +struct WorkloadFixture { + class: CertifiedWorkloadClass, + model_id: String, + model_path: PathBuf, + projector_path: Option, + media_path: Option, + layer_end: u32, + ctx_size: u32, + max_tokens: u32, + n_gpu_layers: i32, +} + +/// Reject absent fixture paths before attempting a model-backed certification. +fn required_file(name: &str) -> Result { + let path = PathBuf::from(env::var_os(name).context(format!("{name} is required"))?); + if !path.is_file() { + bail!("{name} does not point at a file: {}", path.display()); + } + Ok(path) +} + +/// Parse a typed fixture override and report the offending environment key. +fn parse_env(name: &str, default: T) -> Result +where + T: std::str::FromStr, + T::Err: std::fmt::Display, +{ + env::var(name).map_or(Ok(default), |value| { + value + .parse::() + .map_err(|error| anyhow!("parse {name}={value:?}: {error}")) + }) +} + +/// Resolve the explicitly selected class and its required local fixtures. +fn workload_fixture() -> Result> { + let Some(class) = env::var(CLASS_ENV).ok() else { + return Ok(None); + }; + let class = CertifiedWorkloadClass::parse(&class)?; + let projector_path = class + .requires_projector() + .then(|| required_file(PROJECTOR_ENV)) + .transpose()?; + let media_path = class + .requires_media() + .then(|| required_file(MEDIA_ENV)) + .transpose()?; + let model_path = required_file(MODEL_ENV)?; + let layer_end = parse_env(LAYER_END_ENV, 1_u32)?; + if layer_end == 0 { + bail!("{LAYER_END_ENV} must be positive"); + } + Ok(Some(WorkloadFixture { + class, + model_id: env::var(MODEL_ID_ENV).unwrap_or_else(|_| "workload-smoke".to_string()), + model_path, + projector_path, + media_path, + layer_end, + ctx_size: parse_env(CTX_SIZE_ENV, 2048)?, + max_tokens: parse_env(MAX_TOKENS_ENV, 32)?, + n_gpu_layers: parse_env(N_GPU_LAYERS_ENV, 0)?, + })) +} + +/// Build a full-model CPU stage from the selected certification fixture. +fn workload_stage_config(fixture: &WorkloadFixture) -> StageConfig { + StageConfig { + run_id: "workload-certification".to_string(), + topology_id: "workload-certification-local".to_string(), + model_id: fixture.model_id.clone(), + model_path: Some(fixture.model_path.to_string_lossy().to_string()), + projector_path: fixture + .projector_path + .as_ref() + .map(|path| path.to_string_lossy().to_string()), + stage_id: "stage-0".to_string(), + stage_index: 0, + layer_start: 0, + layer_end: fixture.layer_end, + ctx_size: fixture.ctx_size, + lane_count: 1, + n_batch: Some(fixture.ctx_size.min(2048)), + n_ubatch: Some(fixture.ctx_size.min(2048)), + n_gpu_layers: fixture.n_gpu_layers, + kv_offload: (fixture.n_gpu_layers == 0).then_some(false), + op_offload: (fixture.n_gpu_layers == 0).then_some(false), + selected_device: (fixture.n_gpu_layers == 0).then(|| StageDevice { + backend_device: "CPU".to_string(), + stable_id: None, + index: None, + vram_bytes: None, + }), + filter_tensors_on_load: false, + native_mtp_enabled: false, + load_mode: LoadMode::RuntimeSlice, + bind_addr: "127.0.0.1:0".to_string(), + ..StageConfig::default() + } +} + +/// Compare vector dimensions and each component with a bounded numeric tolerance. +fn assert_vectors_close(left: &[f32], right: &[f32]) { + assert_eq!(left.len(), right.len()); + assert!(left.iter().chain(right).all(|value| value.is_finite())); + let maximum_delta = left + .iter() + .zip(right) + .map(|(left, right)| (left - right).abs()) + .fold(0.0_f32, f32::max); + assert!(maximum_delta <= 1e-5, "embedding delta {maximum_delta}"); +} + +/// Distinct inputs must not collapse to one otherwise well-formed embedding vector. +fn require_distinct_embeddings(left: &EmbeddingOutput, right: &EmbeddingOutput) -> Result<()> { + let (EmbeddingOutput::Float(left), EmbeddingOutput::Float(right)) = (left, right) else { + bail!("float embedding request returned a non-float payload"); + }; + anyhow::ensure!(left.len() == right.len(), "embedding dimensions differ"); + anyhow::ensure!( + left.iter().chain(right).all(|value| value.is_finite()), + "embedding contains non-finite values" + ); + anyhow::ensure!( + left.iter() + .zip(right) + .any(|(left, right)| (left - right).abs() > 1e-5), + "distinct inputs returned identical embeddings" + ); + Ok(()) +} + +/// The relevant fixture must strictly outrank the unrelated fixture in response order. +fn require_relevant_document_first(results: &[RerankResult]) -> Result<()> { + anyhow::ensure!( + results.len() == 2 && results[0].index == 0 && results[1].index == 1, + "rerank did not place the relevant document first" + ); + anyhow::ensure!( + results + .iter() + .all(|result| result.relevance_score.is_finite()) + && results[0].relevance_score > results[1].relevance_score, + "the relevant document did not outrank the unrelated document" + ); + Ok(()) +} + +/// A constant normalized vector is not evidence that embeddings depend on their inputs. +#[test] +fn embedding_certification_rejects_constant_vectors() { + let query = EmbeddingOutput::Float(vec![1.0, 0.0]); + assert!(require_distinct_embeddings(&query, &query).is_err()); + assert!(require_distinct_embeddings(&query, &EmbeddingOutput::Float(vec![0.0, 1.0])).is_ok()); +} + +/// Ties, reversed scores, and reordered indexes must not certify a broken ranker. +#[test] +fn rerank_certification_rejects_degenerate_or_misordered_results() { + let mut results = vec![ + RerankResult { + index: 0, + relevance_score: 0.9, + document: None, + }, + RerankResult { + index: 1, + relevance_score: 0.1, + document: None, + }, + ]; + assert!(require_relevant_document_first(&results).is_ok()); + for score in [0.1, 0.0, f32::NAN, f32::INFINITY] { + results[0].relevance_score = score; + assert!(require_relevant_document_first(&results).is_err()); + } + results[0].relevance_score = 0.9; + results.swap(0, 1); + assert!(require_relevant_document_first(&results).is_err()); +} + +/// Check native vector shape, determinism and semantic separation for pinned inputs. +async fn certify_embedding(backend: &StageOpenAiBackend) -> Result<()> { + let info = backend.ensure_local_workload(ModelWorkload::Embedding)?; + assert!(info.output_dimensions > 0); + let request = EmbeddingsRequest { + model: backend.model_id.clone(), + input: EmbeddingInput::Texts(vec![ + "search_query: distributed inference".to_string(), + "search_document: GPUs collaborate over a mesh".to_string(), + ]), + encoding_format: "float".to_string(), + dimensions: None, + user: None, + }; + let first = backend + .embeddings(request.clone(), OpenAiRequestContext::new()) + .await?; + let second = backend + .embeddings(request, OpenAiRequestContext::new()) + .await?; + assert_eq!(first.object, "list"); + assert_eq!(first.data.len(), 2); + assert_eq!(first.data.len(), second.data.len()); + assert!(first.usage.prompt_tokens > 0); + for (left, right) in first.data.iter().zip(&second.data) { + let (EmbeddingOutput::Float(left), EmbeddingOutput::Float(right)) = + (&left.embedding, &right.embedding) + else { + bail!("float embedding request returned a non-float payload"); + }; + assert_eq!(left.len(), info.output_dimensions as usize); + let norm = left.iter().map(|value| value * value).sum::().sqrt(); + assert!((norm - 1.0).abs() <= 1e-4, "embedding norm {norm}"); + assert_vectors_close(left, right); + } + require_distinct_embeddings(&first.data[0].embedding, &first.data[1].embedding)?; + require_distinct_embeddings(&second.data[0].embedding, &second.data[1].embedding)?; + Ok(()) +} + +/// Check deterministic query-document scoring through the workload backend. +async fn certify_rerank(backend: &StageOpenAiBackend) -> Result<()> { + let info = backend.ensure_local_workload(ModelWorkload::Rerank)?; + assert_eq!(info.classifier_outputs, 1); + let request = RerankRequest { + model: backend.model_id.clone(), + query: "distributed GPU inference".to_string(), + documents: vec![ + RerankDocument::Text("GPUs share one language model over a mesh".to_string()), + RerankDocument::Text("A recipe for tomato soup".to_string()), + ], + top_n: None, + return_documents: true, + }; + let first = backend + .rerank(request.clone(), OpenAiRequestContext::new()) + .await?; + let second = backend.rerank(request, OpenAiRequestContext::new()).await?; + assert_eq!(first.results.len(), 2); + assert_eq!(first.results.len(), second.results.len()); + assert!(first.usage.prompt_tokens > 0); + for (left, right) in first.results.iter().zip(&second.results) { + assert_eq!(left.index, right.index); + assert!(left.relevance_score.is_finite()); + assert!((left.relevance_score - right.relevance_score).abs() <= 1e-6); + assert!(left.document.is_some()); + } + require_relevant_document_first(&first.results)?; + require_relevant_document_first(&second.results)?; + Ok(()) +} + +/// Exercise source encoding followed by decoder generation on the pinned model. +async fn certify_encoder_decoder(backend: &StageOpenAiBackend, max_tokens: u32) -> Result<()> { + backend.ensure_local_workload(ModelWorkload::EncoderDecoder)?; + let request: CompletionRequest = serde_json::from_value(json!({ + "model": backend.model_id, + "prompt": "translate English to German: The house is wonderful.", + "max_tokens": max_tokens, + "temperature": 0.0 + }))?; + let first = backend.completion(request.clone()).await?; + let second = backend.completion(request).await?; + assert!(!first.choices[0].text.trim().is_empty()); + assert_eq!(first.choices[0].text, second.choices[0].text); + assert!(first.usage.prompt_tokens > 0); + assert!(first.usage.completion_tokens > 0); + Ok(()) +} + +/// Build an in-process media request from the exact fixture bytes. +fn media_chat_request(fixture: &WorkloadFixture) -> Result { + let path = fixture + .media_path + .as_ref() + .context("media path is required")?; + let encoded = base64::engine::general_purpose::STANDARD.encode(fs::read(path)?); + serde_json::from_value(json!({ + "model": fixture.model_id, + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": "Read all visible text. Return only the transcription."}, + {"type": "image_url", "image_url": {"url": format!("data:image/png;base64,{encoded}")}} + ] + }], + "max_tokens": fixture.max_tokens, + "temperature": 0.0 + })) + .context("build OCR request") +} + +/// Exercise the OCR projector and assert a usable textual result. +async fn certify_ocr(backend: &StageOpenAiBackend, fixture: &WorkloadFixture) -> Result<()> { + let first = backend + .chat_completion(media_chat_request(fixture)?) + .await?; + let second = backend + .chat_completion(media_chat_request(fixture)?) + .await?; + let first_text = first.choices[0] + .message + .content + .as_deref() + .unwrap_or_default() + .trim(); + let second_text = second.choices[0] + .message + .content + .as_deref() + .unwrap_or_default() + .trim(); + assert!(!first_text.is_empty()); + assert_eq!(first_text, second_text); + Ok(()) +} + +/// Verify bounded audio generation and structured unsupported staging behavior. +async fn certify_speech_synthesis(backend: &StageOpenAiBackend) -> Result<()> { + assert!( + backend + .runtime + .lock() + .expect("runtime mutex poisoned") + .supports_speech_synthesis() + ); + let unsupported_voice = backend + .audio_speech( + AudioSpeechRequest { + model: backend.model_id.clone(), + input: "The mesh is ready.".to_string(), + voice: "alloy".to_string(), + response_format: AudioFormat::Wav, + speed: 1.0, + }, + OpenAiRequestContext::new(), + ) + .await + .expect_err("speaker selection must not be interpreted as a language"); + assert_eq!( + unsupported_voice.body().error.code.as_deref(), + Some("unsupported_model_feature") + ); + assert_eq!( + unsupported_voice.body().error.param.as_deref(), + Some("voice") + ); + let response = backend + .audio_speech( + AudioSpeechRequest { + model: backend.model_id.clone(), + input: "The mesh is ready.".to_string(), + voice: "default".to_string(), + response_format: AudioFormat::Wav, + speed: 1.0, + }, + OpenAiRequestContext::new(), + ) + .await?; + assert_eq!(response.content_type, "audio/wav"); + assert!(response.bytes.len() > 44); + assert_eq!(&response.bytes[..4], b"RIFF"); + assert_eq!(&response.bytes[8..12], b"WAVE"); + Ok(()) +} + +/// Exercise transcription with the selected audio and projector fixtures. +async fn certify_speech_recognition( + backend: &StageOpenAiBackend, + fixture: &WorkloadFixture, +) -> Result<()> { + let media_path = fixture + .media_path + .as_ref() + .context("media path is required")?; + let response = backend + .audio_transcription( + AudioTranscriptionRequest { + model: fixture.model_id.clone(), + file: fs::read(media_path)?, + filename: media_path + .file_name() + .map(|name| name.to_string_lossy().to_string()), + language: Some("en".to_string()), + prompt: None, + response_format: "json".to_string(), + temperature: Some(0.0), + }, + OpenAiRequestContext::new(), + ) + .await?; + assert!(!response.text.trim().is_empty()); + Ok(()) +} + +/// Require an explicit rejection when a full-model-only workload is staged. +fn assert_unsupported_staging( + backend: &StageOpenAiBackend, + fixture: &WorkloadFixture, + expected: &str, +) -> Result<()> { + let mut config = workload_stage_config(fixture); + config.filter_tensors_on_load = true; + config.layer_end = (fixture.layer_end / 2).max(1); + config.downstream = Some(PeerConfig { + stage_id: "stage-1".to_string(), + stage_index: 1, + endpoint: "127.0.0.1:1".to_string(), + }); + let runtime = backend.runtime.lock().expect("runtime mutex poisoned"); + let error = match reject_unsupported_staged_workload(&config, &runtime.model) { + Ok(()) => bail!("unsupported workload staging did not fail closed"), + Err(error) => error, + }; + let rendered = format!("{error:#}"); + assert!( + rendered.contains("unsupported staged workload") && rendered.contains(expected), + "unexpected staged workload error: {rendered}" + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +/// Run the selected native class smoke only with its explicit local fixtures. +async fn real_non_chat_class_smoke_when_fixture_is_set() -> Result<()> { + let Some(fixture) = workload_fixture()? else { + return Ok(()); + }; + let backend = + support::local_openai_backend(workload_stage_config(&fixture), fixture.model_id.clone())?; + match fixture.class { + CertifiedWorkloadClass::Embedding => certify_embedding(&backend).await?, + CertifiedWorkloadClass::Rerank => certify_rerank(&backend).await?, + CertifiedWorkloadClass::EncoderDecoder => { + certify_encoder_decoder(&backend, fixture.max_tokens).await? + } + CertifiedWorkloadClass::Ocr => certify_ocr(&backend, &fixture).await?, + CertifiedWorkloadClass::SpeechSynthesis => certify_speech_synthesis(&backend).await?, + CertifiedWorkloadClass::SpeechRecognition => { + certify_speech_recognition(&backend, &fixture).await? + } + } + if let Some(staging_label) = fixture.class.staging_label() { + assert_unsupported_staging(&backend, &fixture, staging_label)?; + } + Ok(()) +} diff --git a/crates/skippy-server/src/frontend/tests/support.rs b/crates/skippy-server/src/frontend/tests/support.rs index 1de05d1a68..a412cea330 100644 --- a/crates/skippy-server/src/frontend/tests/support.rs +++ b/crates/skippy-server/src/frontend/tests/support.rs @@ -117,6 +117,51 @@ pub(super) fn unsupported_code(error: OpenAiError) -> Option { error.body().error.code } +/// Create a real local OpenAI backend for the supplied native model and configuration. +pub(super) fn local_openai_backend( + config: StageConfig, + model_id: impl Into, +) -> Result { + let runtime = load_runtime(&config)?.context("load local test runtime")?; + let ctx_size = usize::try_from(config.ctx_size).unwrap_or(usize::MAX); + let telemetry = Telemetry::new( + None, + 1, + config.clone(), + crate::telemetry::TelemetryLevel::Off, + ); + let iteration_scheduler = + IterationScheduler::new(runtime.clone(), &config, 1, true, telemetry.clone())?; + Ok(StageOpenAiBackend { + runtime, + telemetry, + config, + model_id: model_id.into(), + default_max_tokens: 16, + request_defaults: EmbeddedOpenAiRequestDefaults::default(), + ctx_size, + mode: OpenAiBackendMode::LocalRuntime, + draft: None, + speculative_window: 0, + adaptive_speculative_window: false, + ngram_max: 0, + speculative: SpeculativeDecodeConfig::default(), + generation_limit: Arc::new(GenerationConcurrencyController::fixed(1)), + generation_queue_depth: Arc::new(AtomicUsize::new(0)), + generation_queue_limit: 1, + generation_admission_timeout: Duration::from_secs(10), + generation_service_estimator: Arc::new(GenerationServiceEstimator::new(1)), + generation_session_locks: Arc::new(Mutex::new(std::collections::BTreeMap::new())), + generation_token_budget: Arc::new(GenerationTokenBudget::new(ctx_size)), + hook_policy: None, + generation_receipt: None, + generation_lifecycle: None, + linear_proposal_ingress: None, + kv: None, + iteration_scheduler, + }) +} + pub(super) fn test_request_defaults() -> EmbeddedOpenAiRequestDefaults { EmbeddedOpenAiRequestDefaults { stop: Some(vec!["".to_string()]), diff --git a/crates/skippy-server/src/frontend/tests/tts_oracle.rs b/crates/skippy-server/src/frontend/tests/tts_oracle.rs new file mode 100644 index 0000000000..903ec491e8 --- /dev/null +++ b/crates/skippy-server/src/frontend/tests/tts_oracle.rs @@ -0,0 +1,120 @@ +//! Deterministic test-only TTS output for comparison with pinned llama-tts. +//! +//! The public speech endpoint deliberately uses a random seed. An independent +//! waveform oracle therefore has to drive the same local model execution with +//! fixed sampling parameters rather than compare two random HTTP responses. + +use super::*; +use skippy_runtime::{SpeechOutputFormat, SpeechSynthesisConfig}; + +const OUTPUT_ENV: &str = "SKIPPY_TTS_ORACLE_CANDIDATE_WAV"; + +/// Resolve a required local TTS fixture without downloading replacement inputs. +fn fixture_path(name: &str) -> Result { + let path = PathBuf::from(env::var_os(name).context(format!("{name} is required"))?); + if !path.is_file() { + bail!("{name} does not point at a file: {}", path.display()); + } + Ok(path) +} + +/// Parse deterministic TTS limits from the producer's explicit environment. +fn fixture_number(name: &str) -> Result +where + T: std::str::FromStr, + T::Err: std::fmt::Display, +{ + env::var(name) + .with_context(|| format!("{name} is required"))? + .parse::() + .map_err(|error| anyhow!("parse {name}: {error}")) +} + +/// Configure the CPU candidate to match the reference model and projector setup. +fn local_tts_config( + model_id: &str, + model_path: &Path, + projector_path: &Path, + layer_end: u32, +) -> StageConfig { + StageConfig { + run_id: "tts-monolithic-oracle".to_string(), + topology_id: "tts-monolithic-oracle-local".to_string(), + model_id: model_id.to_string(), + model_path: Some(model_path.to_string_lossy().to_string()), + projector_path: Some(projector_path.to_string_lossy().to_string()), + stage_id: "stage-0".to_string(), + stage_index: 0, + layer_start: 0, + layer_end, + ctx_size: 2048, + lane_count: 1, + n_batch: Some(2048), + n_ubatch: Some(2048), + n_gpu_layers: 0, + kv_offload: Some(false), + op_offload: Some(false), + selected_device: Some(StageDevice { + backend_device: "CPU".to_string(), + stable_id: None, + index: None, + vram_bytes: None, + }), + filter_tensors_on_load: false, + native_mtp_enabled: false, + load_mode: LoadMode::RuntimeSlice, + bind_addr: "127.0.0.1:0".to_string(), + ..StageConfig::default() + } +} + +#[test] +/// Write the complete deterministic candidate waveform for independent comparison. +fn deterministic_tts_candidate_when_fixture_is_set() -> Result<()> { + let Some(output_path) = env::var_os(OUTPUT_ENV) else { + return Ok(()); + }; + let output_path = PathBuf::from(output_path); + let model_path = fixture_path("SKIPPY_WORKLOAD_MODEL")?; + let projector_path = fixture_path("SKIPPY_WORKLOAD_PROJECTOR")?; + let model_id = + env::var("SKIPPY_WORKLOAD_MODEL_ID").context("SKIPPY_WORKLOAD_MODEL_ID is required")?; + let layer_end: u32 = fixture_number("SKIPPY_WORKLOAD_LAYER_END")?; + if layer_end == 0 { + bail!("SKIPPY_WORKLOAD_LAYER_END must be positive"); + } + let prompt = + env::var("SKIPPY_TTS_ORACLE_PROMPT").context("SKIPPY_TTS_ORACLE_PROMPT is required")?; + let seed: u32 = fixture_number("SKIPPY_TTS_ORACLE_SEED")?; + let top_k: i32 = fixture_number("SKIPPY_TTS_ORACLE_TOP_K")?; + let top_p: f32 = fixture_number("SKIPPY_TTS_ORACLE_TOP_P")?; + let max_frames: usize = fixture_number("SKIPPY_TTS_ORACLE_MAX_FRAMES")?; + if prompt.is_empty() || top_k < 1 || !(0.0..=1.0).contains(&top_p) || max_frames == 0 { + bail!("invalid deterministic TTS oracle parameters"); + } + + let config = local_tts_config(&model_id, &model_path, &projector_path, layer_end); + let runtime = load_runtime(&config)?.context("load full-model TTS oracle candidate")?; + let mut runtime = runtime + .lock() + .map_err(|_| anyhow!("TTS oracle candidate runtime lock poisoned"))?; + let audio = runtime.synthesize_speech( + "tts-oracle", + &SpeechSynthesisConfig { + prompt, + language: None, + top_k, + top_p, + seed, + output_format: SpeechOutputFormat::Wav, + max_frames, + }, + || false, + )?; + if audio.bytes.len() <= 44 || audio.sample_rate == 0 || audio.sample_count == 0 { + bail!("deterministic TTS candidate produced no WAV samples"); + } + fs::write(&output_path, &audio.bytes) + .with_context(|| format!("write candidate WAV to {}", output_path.display()))?; + Ok(()) +} diff --git a/crates/skippy-server/src/http.rs b/crates/skippy-server/src/http.rs index 0a399068cd..2cf91f9dbd 100644 --- a/crates/skippy-server/src/http.rs +++ b/crates/skippy-server/src/http.rs @@ -51,6 +51,13 @@ pub(crate) fn bind_serve_listener(bind_addr: SocketAddr) -> Result }; let socket = socket2::Socket::new(domain, socket2::Type::STREAM, Some(socket2::Protocol::TCP)) .with_context(|| format!("create serving socket for {bind_addr}"))?; + // Match Tokio's Unix listener semantics: a stopped server may rebind while + // its closed connections are in TIME_WAIT. Do not enable SO_REUSEPORT or + // Windows SO_REUSEADDR, which can permit sharing a live listener's port. + #[cfg(unix)] + socket + .set_reuse_address(true) + .context("enable serving listener address reuse")?; socket .bind(&bind_addr.into()) .with_context(|| format!("bind serving socket to {bind_addr}"))?; @@ -809,6 +816,44 @@ mod tests { use super::*; + #[cfg(unix)] + #[tokio::test] + /// Verify a completed server can immediately reuse its listening address. + async fn serving_listener_can_rebind_after_server_closes_connection() -> Result<()> { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + tokio::time::timeout(std::time::Duration::from_secs(5), async { + let listener = bind_serve_listener("127.0.0.1:0".parse()?)?; + let addr = listener.local_addr()?; + let mut client = tokio::net::TcpStream::connect(addr).await?; + let (mut accepted, _) = listener.accept().await?; + // The server actively closes, leaving its connection in TIME_WAIT. + accepted.shutdown().await?; + client.read_to_end(&mut Vec::new()).await?; + drop(accepted); + drop(client); + drop(listener); + let replacement = bind_serve_listener(addr)?; + assert_eq!(replacement.local_addr()?, addr); + Ok::<_, anyhow::Error>(()) + }) + .await + .context("serving listener restart timed out")? + } + + #[tokio::test] + /// Preserve exclusive address ownership while the first listener remains live. + async fn serving_listener_rejects_another_live_listener() -> Result<()> { + let listener = bind_serve_listener("127.0.0.1:0".parse()?)?; + let error = bind_serve_listener(listener.local_addr()?) + .expect_err("a second listener must not share the live port"); + assert_eq!( + error.downcast_ref::().unwrap().kind(), + std::io::ErrorKind::AddrInUse + ); + Ok(()) + } + fn stage_config_without_runtime() -> StageConfig { StageConfig { run_id: "run".to_owned(), diff --git a/crates/skippy-server/src/runtime_state.rs b/crates/skippy-server/src/runtime_state.rs index c184c3680b..b0c556208a 100644 --- a/crates/skippy-server/src/runtime_state.rs +++ b/crates/skippy-server/src/runtime_state.rs @@ -11,9 +11,10 @@ use skippy_runtime::{ DecodeFrameBatchRequest, FlashAttentionType as RuntimeFlashAttentionType, GenerationSignalWindow, GlmDsaPolicy as RuntimeGlmDsaPolicy, IterationBatchOutput, IterationBatchPhase, IterationBatchRequest, MediaInput, MediaPrefill, MediaPrefillFrame, - ModelStateKind, MtpSource, NativeMtpDraft, RuntimeConfig, RuntimeKvPage, RuntimeKvPageDesc, - RuntimeLoadMode, SamplingConfig, SplitMode as RuntimeSplitMode, StageModel, StageSession, - TokenSignal, parse_cache_type, + ModelStateKind, ModelWorkload, MtpSource, NativeMtpDraft, RuntimeConfig, RuntimeKvPage, + RuntimeKvPageDesc, RuntimeLoadMode, SamplingConfig, SpeechAudio, SpeechSynthesisConfig, + SplitMode as RuntimeSplitMode, StageModel, StageSession, TokenSignal, WorkloadInfo, + parse_cache_type, }; mod frame_operations; @@ -146,6 +147,48 @@ struct ResidentLanePrefix { } impl RuntimeState { + /// Read the workload descriptor from the model actually loaded into this runtime. + pub fn workload_info(&self) -> Result { + self.model.workload_info() + } + + /// Execute one tokenized embedding request in the caller's owned session. + pub fn embed( + &mut self, + session_id: &str, + token_ids: &[i32], + dimensions: usize, + ) -> Result> { + let embedding = self.session(session_id)?.embed(token_ids, dimensions)?; + self.session_token_counts.insert( + session_id.to_string(), + u64::try_from(token_ids.len()).context("embedding token count exceeds u64")?, + ); + Ok(embedding) + } + + /// Score one query-document pair and update its session's token accounting. + pub fn rerank( + &mut self, + session_id: &str, + query: &str, + document: &str, + ) -> Result<(f32, usize)> { + let result = self.session(session_id)?.rerank(query, document)?; + self.session_token_counts.insert( + session_id.to_string(), + u64::try_from(result.1).context("rerank token count exceeds u64")?, + ); + Ok(result) + } + + /// Encode a source prompt and initialize decoder-session token accounting. + pub fn encode_prompt(&mut self, session_id: &str, token_ids: &[i32]) -> Result { + let decoder_start = self.session(session_id)?.encode_prompt(token_ids)?; + self.session_token_counts.insert(session_id.to_string(), 0); + Ok(decoder_start) + } + pub fn input_activation_boundary(&self) -> Option { self.model.input_activation_boundary() } @@ -286,8 +329,21 @@ pub fn load_runtime_with_overrides( open_stage_model(model_path, &runtime_config)? } }; + Ok(Some(runtime_from_loaded_model( + config, + model, + session_lifecycle_observer, + )?)) +} - Ok(Some(Arc::new(Mutex::new(RuntimeState { +/// Apply workload admission and retain the host's session-lifecycle observer. +fn runtime_from_loaded_model( + config: &StageConfig, + model: StageModel, + session_lifecycle_observer: Option>, +) -> Result>> { + reject_unsupported_staged_workload(config, &model)?; + Ok(Arc::new(Mutex::new(RuntimeState { model, layer_start: config.layer_start, layer_end: config.layer_end, @@ -303,7 +359,7 @@ pub fn load_runtime_with_overrides( session_lifecycle_observer, #[cfg(test)] modelless_for_test: false, - })))) + }))) } pub fn load_runtime_with_overrides_and_open_events( @@ -343,24 +399,39 @@ pub fn load_runtime_with_overrides_and_open_events( )? } }; - - Ok(Some(Arc::new(Mutex::new(RuntimeState { + Ok(Some(runtime_from_loaded_model( + config, model, - layer_start: config.layer_start, - layer_end: config.layer_end, - lane_count: config.lane_count, - ctx_size: config.ctx_size, - next_lane_index: 0, - free_lane_indices: Vec::new(), - sessions: BTreeMap::new(), - idle_sessions: Vec::new(), - max_idle_sessions: max_idle_sessions_from_stage_config(config), - session_token_counts: BTreeMap::new(), - session_resident_prefixes: BTreeMap::new(), session_lifecycle_observer, - #[cfg(test)] - modelless_for_test: false, - })))) + )?)) +} + +/// Reject full-model-only workloads before filtered stages can begin serving. +pub(crate) fn reject_unsupported_staged_workload( + config: &StageConfig, + model: &StageModel, +) -> Result<()> { + if !config.filter_tensors_on_load { + return Ok(()); + } + if model.supports_speech_synthesis() { + anyhow::bail!( + "unsupported staged workload speech_synthesis: audio generation requires an unsplit full model" + ); + } + let workload = model.workload_info()?.kind; + if workload != ModelWorkload::CausalGeneration { + anyhow::bail!( + "unsupported staged workload {}: non-chat execution requires an unsplit full model", + match workload { + ModelWorkload::CausalGeneration => unreachable!(), + ModelWorkload::Embedding => "embedding", + ModelWorkload::Rerank => "rerank", + ModelWorkload::EncoderDecoder => "encoder_decoder", + } + ); + } + Ok(()) } /// Translates `model_fit.cache_idle_slots` into the idle-session-pool bound. diff --git a/crates/skippy-server/src/runtime_state/frame_operations.rs b/crates/skippy-server/src/runtime_state/frame_operations.rs index e8155cfa67..832100333b 100644 --- a/crates/skippy-server/src/runtime_state/frame_operations.rs +++ b/crates/skippy-server/src/runtime_state/frame_operations.rs @@ -44,6 +44,25 @@ impl RuntimeState { self.model.has_media_projector() } + /// Probe speech support from the runtime's actual native model and projector. + pub fn supports_speech_synthesis(&self) -> bool { + self.model.supports_speech_synthesis() + } + + /// Run bounded speech generation in the named session with cancellation checks. + pub fn synthesize_speech( + &mut self, + session_id: &str, + config: &SpeechSynthesisConfig, + cancellation_requested: impl Fn() -> bool, + ) -> Result { + let model = &self.model as *const StageModel; + let session = self.session(session_id)?; + // The outer RuntimeState mutex serializes both projector and session + // access; this splits borrows across those independently owned fields. + unsafe { (&*model).synthesize_speech(session, config, cancellation_requested) } + } + pub fn prefill_media( &mut self, session_id: &str, diff --git a/crates/skippy-server/src/runtime_state/lane_lifecycle/tests.rs b/crates/skippy-server/src/runtime_state/lane_lifecycle/tests.rs index c126c5783a..d23a31c88d 100644 --- a/crates/skippy-server/src/runtime_state/lane_lifecycle/tests.rs +++ b/crates/skippy-server/src/runtime_state/lane_lifecycle/tests.rs @@ -16,6 +16,27 @@ impl crate::runtime_state::SessionLifecycleObserver for RecordingSessionObserver } } +#[test] +/// Both model-loading paths share this constructor; admission must retain the +/// observer supplied by the host instead of silently dropping lifecycle events. +fn admitted_runtime_preserves_session_lifecycle_observer() -> Result<()> { + let observer = Arc::new(RecordingSessionObserver::default()); + let runtime = crate::runtime_state::runtime_from_loaded_model( + &skippy_protocol::StageConfig::default(), + skippy_runtime::StageModel::new_dummy(), + Some(observer.clone()), + )?; + runtime + .lock() + .unwrap() + .notify_session_lifecycle(crate::runtime_state::SessionLifecycleEvent::SessionReclaimed); + assert_eq!( + *observer.0.lock().unwrap(), + vec![crate::runtime_state::SessionLifecycleEvent::SessionReclaimed,] + ); + Ok(()) +} + #[test] fn dropping_an_absent_session_notifies_nothing() { let observer = Arc::new(RecordingSessionObserver::default()); diff --git a/docs/NON_CHAT_MODELS.md b/docs/NON_CHAT_MODELS.md new file mode 100644 index 0000000000..a1ebc027ff --- /dev/null +++ b/docs/NON_CHAT_MODELS.md @@ -0,0 +1,222 @@ +# Non-Chat Model Workloads + +Mesh LLM can serve several llama.cpp model classes whose primary contract is +not causal chat generation. The public OpenAI-compatible endpoint stays on the +normal mesh port (`http://127.0.0.1:9337/v1`), while the loaded model advertises +its runtime-probed workload class for safe routing. + +## Supported surfaces + +| Model class | HTTP surface | Current execution boundary | +|---|---|---| +| Embedding / encoder-only | `POST /v1/embeddings` | Unsplit local full model | +| Cross-encoder rerank | `POST /v1/rerank` | Unsplit local full model | +| Encoder-decoder | `POST /v1/completions`, `POST /v1/chat/completions` | Unsplit local full model | +| OCR | `POST /v1/chat/completions` or `POST /v1/responses` with an image input | Causal trunk and projector colocated on one node | +| Speech synthesis | `POST /v1/audio/speech` | Unsplit local full model and projector | +| Speech recognition / translation | `POST /v1/audio/transcriptions`, `POST /v1/audio/translations` | Causal trunk and audio projector colocated on one node | + +Embedding, rerank, encoder-decoder, and speech-synthesis workloads do not +silently enter the stage-split generation path. A filtered stage model returns +a structured `unsupported` error. OCR and speech recognition use causal trunks, +but their projector remains local to the trunk; they are not claims of a +distributed projector implementation. + +## Mixed-version routing + +A model name can be advertised by several nodes running different versions. +Model discovery accepts any compatible advertisement, but each serving target +must independently advertise the workload required by the endpoint. The host +router and passive-client proxy exclude incompatible targets before context +ranking, cache affinity, reservation spreading, and retries. One current peer +does not grant its endpoints to a legacy peer serving the same model name. + +Absent workload metadata remains compatible with ordinary generation requests, +not with embedding, rerank, speech synthesis, or audio-upload endpoints. Audio +uploads additionally require runtime-verified audio support in the same target's +model descriptor. Unsupported targets are not restored by an availability +fallback or a cached automatic model choice. + +## Embeddings + +The request accepts a string, an array of strings, a token array, or an array of +token arrays. `encoding_format` may be `float` or `base64`. Base64 values encode +little-endian `f32` values. The runtime honors the pooling type recorded in the +GGUF and returns L2-normalized vectors. + +```python +from openai import OpenAI + +client = OpenAI(api_key="mesh", base_url="http://127.0.0.1:9337/v1") +response = client.embeddings.create( + model="nomic-embed-text-v1.5-Q8_0", + input=["search_query: distributed inference", "search_document: pooled GPUs"], + encoding_format="float", +) +print(len(response.data[0].embedding)) +``` + +The optional `dimensions` field is accepted only when it equals the model's +native output width. Dimensionality reduction is not performed implicitly. + +## Reranking + +`/v1/rerank` is the common cross-encoder companion surface. Documents may be +strings or objects containing a string `text` field. + +```bash +curl -sS http://127.0.0.1:9337/v1/rerank \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "jina-reranker-v1-turbo-en", + "query": "distributed model inference", + "documents": ["one GPU", "several GPUs connected over a mesh"], + "top_n": 2, + "return_documents": true + }' +``` + +Results are ordered by descending relevance score. Each result retains the +zero-based index of the original document. + +## Encoder-decoder generation and OCR + +Encoder-decoder GGUFs use the existing completion and chat response shapes. +Their encoder pass and decoder loop are executed by the local Skippy runtime; +tool calls are rejected because the current encoder-decoder path does not +implement that contract. + +OCR remains a multimodal text-generation request. Send the image using the +existing `image_url`/`input_image` content-part contract. Automatic routing +requires a model that advertises both a causal workload and vision capability. + +## Audio + +Speech synthesis returns raw binary data with the matching response content +type. The current native Qwen3-TTS path supports `wav` (`audio/wav`) and `pcm` +(`audio/pcm`). Other OpenAI response formats are parsed and rejected with a +structured unsupported error instead of returning bytes under the wrong media +type. Set `response_format` explicitly because the OpenAI request default is +`mp3`, which this native path does not encode. + +```bash +curl -sS http://127.0.0.1:9337/v1/audio/speech \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "Qwen3-TTS-12Hz-1.7B-Base-Q8_0", + "input": "The mesh is ready.", + "voice": "default", + "response_format": "wav" + }' \ + --output speech.wav +``` + +Transcription and translation accept `multipart/form-data` with a required +`model` field and one `file` field. Uploads are bounded at 64 MiB. The supported +response formats are `json` and `text`. + +```bash +curl -sS http://127.0.0.1:9337/v1/audio/transcriptions \ + -F model=ultravox-v0_5-llama-3_2-1b \ + -F file=@recording.wav \ + -F response_format=json +``` + +Automatic audio routing filters for a runtime-verified audio capability. An +explicitly selected incompatible model is rejected by the local backend rather +than silently discarding the audio. + +## Class-specific workload certification + +Every row in `ci/llama-canary/family-certified.json` records an explicit +`class`. The six checked-in non-chat rows use the certified `workload-oracle` +profile with separate class-specific smoke and local-monolithic oracle lanes; +they do not inherit the causal-generation `full` profile or its three split +handoff lanes. The checked-in representatives cover +embedding, rerank, encoder-decoder, OCR, speech synthesis, and speech recognition +with immutable Hugging Face revisions and artifact digests. + +Run a focused local smoke with the model files already present: + +```bash +scripts/skippy-workload-certify.sh \ + --class embedding \ + --lane embedding-smoke \ + --model-path /path/to/model.gguf \ + --model-id local-embedding \ + --work-dir target/workload-certification +``` + +Projector-backed classes additionally require `--projector-path`. Each lane +checks local real-model behavior and exercises its HTTP endpoint through the +OpenAI frontend. The embedding lane also uses the official Python OpenAI SDK +when that optional package is installed; its mandatory HTTP assertions do not +depend on the SDK being present. + +An isolated invocation without an oracle remains a smoke check only: it verifies +local execution, the HTTP response contract, and class-specific coarse +assertions, but cannot satisfy the checked-in certified profile. Filtered +staged workloads also fail closed where unsupported. The mandatory family +battery oracle uses test-only llama.cpp executables built from the same pinned +patch queue; those executables are never packaged as Mesh-LLM serving backends: + +```bash +LLAMA_STAGE_BACKEND=cpu \ +LLAMA_STAGE_LINK_MODE=static \ +LLAMA_STAGE_BUILD_DIR=/path/to/candidate-static-build \ +just llama-build + +LLAMA_STAGE_BACKEND=cpu \ +LLAMA_STAGE_LINK_MODE=static \ +LLAMA_STAGE_BUILD_DIR=/path/to/isolated-oracle-build \ +LLAMA_STAGE_WORKLOAD_ORACLE=ON \ +just llama-build + +LLAMA_STAGE_BACKEND=cpu \ +LLAMA_STAGE_LINK_MODE=static \ +LLAMA_STAGE_BUILD_DIR=/path/to/candidate-static-build \ +scripts/skippy-workload-certify.sh \ + --class embedding \ + --lane embedding-smoke \ + --model-path /path/to/model.gguf \ + --model-id local-embedding \ + --work-dir target/workload-certification \ + --oracle-server /path/to/isolated-oracle-build/bin/llama-server \ + --require-oracle +``` + +The runner checks each oracle executable's current pinned-source build stamp +and requires CPU-only execution, including Metal disabled on macOS. The +monolithic reference uses `--no-repack` to match the staged runtime's default +model-loading configuration; omitting that flag can create a numerical +mismatch unrelated to the workload implementation. Use +`--oracle-server` for embedding, rerank, OCR, or speech recognition; +`--oracle-completion` with `bin/llama-completion` for encoder-decoder; and +`--oracle-tts` with `bin/llama-tts` for speech synthesis. The family battery +accepts the corresponding `SKIPPY_WORKLOAD_ORACLE_SERVER`, +`SKIPPY_WORKLOAD_ORACLE_COMPLETION`, and `SKIPPY_WORKLOAD_ORACLE_TTS` +environment variables; a certified family fails closed if its class-appropriate +executable is absent. The direct completion CLI is intentional: the pinned +`llama-server` completion endpoint does not produce the correct FLAN-T5 text +for this fixture, while the pinned monolithic `llama-completion` encoder pass +and decoder loop do. + +The per-class oracle gates are: + +| Class | Independent comparison | Important limit | +|---|---|---| +| Embedding | Identical batch and each individual input; same vector width, maximum coordinate error `1e-4`, minimum cosine `0.99999` | Parity does not measure retrieval quality | +| Rerank | Identical query/documents; maximum score error `1e-4` and identical ordering | Parity does not measure ranking quality | +| Encoder-decoder | Same prompt and greedy seed; identical text after whitespace normalization | Pinned direct monolithic completion CLI, not the server endpoint | +| OCR | Same generated `MESH 42` PNG and prompt; normalized text matches and contains the independently known fixture label | One synthetic image does not certify broad OCR accuracy | +| Speech recognition | Same WAV and prompt; normalized text matches | The generic smoke WAV has no checked-in transcript label, so this is execution parity, not transcription accuracy | +| Speech synthesis | Same prompt, seed, top-k/top-p, and frame cap in deterministic in-process Skippy and monolithic `llama-tts`; PCM format/length match, relative RMS error at most `2%`, waveform cosine at least `0.9995` | Public HTTP speech sampling is stochastic; PCM parity does not establish intelligibility | + +The family battery records an oracle pass only when the class-specific +comparison completes and a run-local evidence file matches the pinned model, +projector, candidate executable, oracle executable, and patch SHA. A smoke +pass cannot substitute for that evidence. The original provisional +`workload-smoke` profile remains available for future rows that have not yet +passed an independent oracle. Certification here is a bounded equivalence +claim for these six pinned artifacts and fixtures, not broad model-quality +or split-serving certification. diff --git a/docs/README.md b/docs/README.md index 238750fe06..a50f1c9387 100644 --- a/docs/README.md +++ b/docs/README.md @@ -14,6 +14,7 @@ Use this hub to find project guides that are not owned by a single Rust crate. | SDK usage, examples, errors, lifecycle, platform support | [SDK.md](SDK.md) | | Language-specific SDK examples | [Rust](sdk/rust.md), [Node.js](sdk/node.md), [Swift](sdk/swift.md), [Kotlin/Android](sdk/kotlin.md) | | Run big models with Skippy layer splits | [SKIPPY_SPLITS.md](SKIPPY_SPLITS.md) | +| Embedding, rerank, encoder-decoder, OCR, and audio models | [NON_CHAT_MODELS.md](NON_CHAT_MODELS.md) | | Contribute or publish layer package repositories | [LAYER_PACKAGE_REPOS.md](LAYER_PACKAGE_REPOS.md) | | Goose, Claude Code, OpenCode, Pi, curl, blackboard | [AGENTS.md](AGENTS.md) | | Command-by-command CLI reference | [CLI.md](CLI.md) | diff --git a/docs/SKIPPY.md b/docs/SKIPPY.md index b5af69e0ef..9b78559087 100644 --- a/docs/SKIPPY.md +++ b/docs/SKIPPY.md @@ -754,14 +754,21 @@ router it provides. Mesh's public ingress should be able to: - stream responses through the shared adapters without buffering the full response body. -Embeddings can be deferred for now. The replacement plan does not need -`/v1/embeddings` parity before removing `llama-server`. +The shared frontend and local full-model runtime now cover embedding, rerank, +encoder-decoder, OCR, speech-synthesis, and speech-recognition workloads. These +paths are intentionally separate from staged causal generation: unsupported +stage shapes return structured errors instead of inheriting a generation lane. +See [NON_CHAT_MODELS.md](NON_CHAT_MODELS.md) for the current endpoint and +certification matrix. Current branch status: - `openai-frontend` owns `/v1/chat/completions`, `/v1/completions`, and `/v1/responses` request/response shapes, streaming SSE adapters, OpenAI error bodies, tool-call fields, structured-output fields, and logprob fields; +- `openai-frontend` also owns `/v1/embeddings`, `/v1/rerank`, and `/v1/audio/*` + request/response shapes, including bounded multipart uploads and binary audio + responses; - frontend fixture coverage accepts and translates tool, structured-output, logprob, streaming, and responses requests without mesh carrying a second public OpenAI model; diff --git a/docs/design/MULTI_MODAL.md b/docs/design/MULTI_MODAL.md index 67a7e695b0..2a4adade12 100644 --- a/docs/design/MULTI_MODAL.md +++ b/docs/design/MULTI_MODAL.md @@ -12,7 +12,10 @@ Phases 1 through 5 in this plan are complete: - multimodal `/v1/chat/completions` - multimodal `/v1/responses` for both non-streaming and streaming requests -What remains is polish and hardening of the existing path, not new endpoint scope. +The original chat/Responses phases are complete. Full-model audio endpoints +and OCR family certification are documented separately in +[NON_CHAT_MODELS.md](../NON_CHAT_MODELS.md); projector execution remains +colocated with its model trunk. ## Goals @@ -26,10 +29,8 @@ What remains is polish and hardening of the existing path, not new endpoint scop - Permanent distributed file storage - IPFS/libp2p-first design -- `POST /v1/audio/transcriptions` -- `POST /v1/audio/speech` - `v1/realtime` -- Audio generation from llama alone +- Distributed projector execution - Native end-to-end video inference on the current llama.cpp path ## API Targets @@ -47,6 +48,15 @@ This is the shortest path because llama.cpp already supports multimodal chat her Implement as a mesh-llm compatibility shim after chat completions are solid. +### Full-model audio extensions + +- `POST /v1/audio/speech` +- `POST /v1/audio/transcriptions` +- `POST /v1/audio/translations` + +These routes are available for runtime-compatible local model/projector pairs. +They do not imply a split projector or distributed audio-generation graph. + ## Capability Model Do not collapse everything into `vision`. diff --git a/docs/design/TESTING.md b/docs/design/TESTING.md index 7bb15a14c3..fe325655b1 100644 --- a/docs/design/TESTING.md +++ b/docs/design/TESTING.md @@ -324,10 +324,11 @@ seed/step trace in its uploaded log. Repository variables set `MESH_NIGHTLY_KV_COVERAGE_ENABLED=0` to disable the scheduled run. Manual dispatch still executes trusted `main` on GitHub-hosted infrastructure. -The unchanged-pin daily llama canary uses the `nightly` cadence in -`ci/llama-canary/family-certified.json`: Qwen3 dense, Falcon-H1 hybrid, -Qwen3Next composite, and Mamba recurrent. Llama bumps and explicit forced -certification retain the full 33-family battery. +The daily llama canary, pin advances, and explicit forced certification all +use the complete `ci/llama-canary/family-certified.json` roster: 83 causal +split targets plus six non-chat workloads. Trigger labels do not filter model +coverage. Causal rows use product-approved topology cuts; non-chat rows require +class-specific smoke and independent CPU oracle evidence, never split proof. ### 0g. Logging workflow certification diff --git a/just/ci.just b/just/ci.just index fe36f40e0f..3a58407719 100644 --- a/just/ci.just +++ b/just/ci.just @@ -109,8 +109,13 @@ test-all: --exclude mesh-llm-nodejs \ --exclude skippy-ffi \ --exclude skippy-quantize \ + --exclude skippy-model-package \ --exclude skippy-runtime \ --exclude skippy-server + # The workspace's dynamic-native-runtime feature otherwise unifies into + # the package fixture tests, which require the statically linked GGUF ABI. + echo "--- Static Skippy model-package tests ---" + just with-lld cargo test --package skippy-model-package --bin skippy-model-package echo "--- Static Skippy runtime tests ---" just with-lld cargo test --package skippy-runtime --no-default-features --lib echo "--- Static Skippy server tests ---" diff --git a/just/skippy.just b/just/skippy.just index 43fa0b683a..f5dff0a53f 100644 --- a/just/skippy.just +++ b/just/skippy.just @@ -44,6 +44,11 @@ skippy-quantize-standalone-release-build backend="cpu": LLAMA_STAGE_BACKEND="{{ backend }}" LLAMA_STAGE_LINK_MODE=static scripts/build-llama.sh LLAMA_STAGE_BACKEND="{{ backend }}" LLAMA_STAGE_LINK_MODE=static just with-lld cargo build --release --locked -p skippy-quantize --no-default-features +# Build isolated CPU workload certification producers (not a shipping runtime). +[unix] +skippy-workload-oracles-build output: + bash scripts/skippy-workload-oracles-build.sh "{{ output }}" + # Build and execute the opt-in private native Skippy application probes. [unix] skippy-native-tests backend="cpu": diff --git a/scripts/build-llama.sh b/scripts/build-llama.sh index 006cfe4c44..f1ca693bbc 100755 --- a/scripts/build-llama.sh +++ b/scripts/build-llama.sh @@ -13,6 +13,7 @@ LLAMA_LINK_MODE="${LLAMA_STAGE_LINK_MODE:-${SKIPPY_LLAMA_LINK_MODE:-static}}" LLAMA_STAGE_BUILD_TESTS="${LLAMA_STAGE_BUILD_TESTS:-OFF}" LLAMA_STAGE_FULL_REPLAY="${LLAMA_STAGE_FULL_REPLAY:-OFF}" LLAMA_STAGE_UPSTREAM_TESTS="${LLAMA_STAGE_UPSTREAM_TESTS:-OFF}" +LLAMA_STAGE_WORKLOAD_ORACLE="${LLAMA_STAGE_WORKLOAD_ORACLE:-OFF}" LLAMA_BUILD_TESTS=OFF LLAMA_BUILD_SERVER=OFF if [[ "$LLAMA_STAGE_FULL_REPLAY" == "ON" || "$LLAMA_STAGE_UPSTREAM_TESTS" == "ON" ]]; then @@ -24,6 +25,9 @@ if [[ "$LLAMA_STAGE_UPSTREAM_TESTS" == "ON" ]]; then # multimodal include/link closure. LLAMA_BUILD_SERVER=ON fi +if [[ "$LLAMA_STAGE_WORKLOAD_ORACLE" == "ON" ]]; then + LLAMA_BUILD_SERVER=ON +fi PRINT_BUILD_DIR=0 REQUIRE_EXISTING=0 @@ -151,9 +155,15 @@ required_dynamic_libraries_exist() { required_outputs_exist() { if [[ "$LLAMA_LINK_MODE" == "dynamic" ]]; then - required_dynamic_libraries_exist + required_dynamic_libraries_exist || return 1 else - required_static_archives_exist + required_static_archives_exist || return 1 + fi + if [[ "$LLAMA_STAGE_WORKLOAD_ORACLE" == "ON" ]]; then + [[ -x "$LLAMA_BUILD_DIR/bin/llama-server" && + -x "$LLAMA_BUILD_DIR/bin/llama-cli" && + -x "$LLAMA_BUILD_DIR/bin/llama-completion" && + -x "$LLAMA_BUILD_DIR/bin/llama-tts" ]] || return 1 fi } @@ -220,6 +230,14 @@ CMAKE_ARGS=( # platforms. -DMTMD_VIDEO=OFF ) +if [[ "$LLAMA_STAGE_WORKLOAD_ORACLE" == "ON" ]]; then + CMAKE_ARGS+=(-DLLAMA_BUILD_COMMON=ON -DLLAMA_BUILD_TOOLS=ON) +fi +if [[ "$LLAMA_BACKEND" == "cpu" ]]; then + # macOS defaults Metal to ON even when the selected backend is CPU. Match + # the backend contract for both the embedded runtime and its test oracle. + CMAKE_ARGS+=(-DGGML_METAL=OFF) +fi # Static ABI inputs cross job and runner boundaries. Normalize compiler- # embedded source/build paths so the archived link closure does not retain a @@ -406,6 +424,10 @@ fi cmake "${CMAKE_ARGS[@]}" BUILD_TARGETS=(llama llama-common mtmd) +if [[ "$LLAMA_STAGE_WORKLOAD_ORACLE" == "ON" ]]; then + # Test-only full-model references. They are never packaged beside the host. + BUILD_TARGETS+=(llama-server llama-cli llama-completion llama-tts) +fi if [[ "$LLAMA_STAGE_BUILD_TESTS" == "ON" ]]; then BUILD_TARGETS+=( skippy-graph-build-inputs @@ -425,6 +447,8 @@ if [[ "$LLAMA_STAGE_FULL_REPLAY" == "ON" ]]; then test-skippy-kv-page-export test-skippy-model-loader-accounting test-skippy-recurrent-state-roundtrip + test-skippy-rerank-template + test-skippy-sampling-suppress test-skippy-verify-checkpoint-retirement ) fi diff --git a/scripts/check-skippy-workload-candidate.py b/scripts/check-skippy-workload-candidate.py new file mode 100644 index 0000000000..e727992e34 --- /dev/null +++ b/scripts/check-skippy-workload-candidate.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Reject a stale statically linked candidate before an oracle comparison.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess + +ROOT = Path(__file__).resolve().parents[1] + + +def file_hash(path: Path) -> str: + """Hash artifact bytes incrementally without loading model-sized files in memory.""" + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def source_identity(root: Path = ROOT) -> dict[str, str]: + """Bind the checkout head and all nonignored source changes to one identity.""" + def git(*args: str) -> bytes: + """Read repository state without changing the checkout or index.""" + return subprocess.check_output(["git", "-C", str(root), *args]) + + digest = hashlib.sha256(git("diff", "--binary", "HEAD", "--")) + # Bind new source files as well as staged/unstaged tracked changes. Build + # outputs are ignored by Git and cannot change this identity during a run. + for name in sorted(git("ls-files", "--others", "--exclude-standard", "-z").split(b"\0")): + if name: + digest.update(name + b"\0") + digest.update(file_hash(root / os.fsdecode(name)).encode("ascii")) + return {"head": git("rev-parse", "HEAD").decode().strip(), "worktree_sha256": digest.hexdigest()} + + +def producer_files(binary: Path, build_dir: Path, test_binary: Path) -> dict[str, Path]: + """Enumerate the candidate, native stamp, tools, and independent oracle closure.""" + return { + "candidate": binary, + "test_binary": test_binary, + "model_package": binary.parent / "skippy-model-package", + "correctness": binary.parent / "skippy-correctness", + "topology_plan": binary.parent / "skippy-topology-plan", + "native_stamp": build_dir / ".mesh-llm-build-stamp", + **{name: build_dir / "bin" / name for name in ("llama-server", "llama-completion", "llama-tts")}, + } + + +def write_producer(output: Path, binary: Path, build_dir: Path, test_binary: Path, source_snapshot: Path) -> None: + """Write evidence only if source stayed unchanged throughout the producer build.""" + source = source_identity() + if source != json.loads(source_snapshot.read_text(encoding="utf-8")): + raise RuntimeError("repository source changed while building workload producers") + check_candidate(binary, build_dir) + check_candidate(test_binary, build_dir) + files = producer_files(binary, build_dir, test_binary) + payload = { + "schema_version": 1, + "source": source, + "files": {name: {"path": str(path.resolve()), "sha256": file_hash(path)} for name, path in files.items()}, + } + output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def verify_producer(manifest: Path, binary: Path, build_dir: Path) -> None: + """Reject stale source, replaced executables, or a mismatched native test closure.""" + payload = json.loads(manifest.read_text(encoding="utf-8")) + if payload.get("schema_version") != 1 or payload.get("source") != source_identity(): + raise RuntimeError("workload producer does not match the current repository head and worktree") + records = payload["files"] + files = producer_files(binary, build_dir, Path(records["test_binary"]["path"])) + for name, path in files.items(): + record = records[name] + if record["path"] != str(path.resolve()) or record["sha256"] != file_hash(path): + raise RuntimeError(f"workload producer artifact changed: {name}") + check_candidate(files["test_binary"], build_dir) + + +def check_candidate(binary: Path, build_dir: Path) -> None: + """Require an executable newer than its stamped, statically linked native ABI.""" + stamp = build_dir / ".mesh-llm-build-stamp" + if not binary.is_file() or not os.access(binary, os.X_OK): + raise RuntimeError(f"candidate executable is missing: {binary}") + if not stamp.is_file(): + raise RuntimeError(f"candidate native build stamp is missing: {stamp}") + if binary.stat().st_mtime_ns <= stamp.stat().st_mtime_ns: + raise RuntimeError( + "candidate executable predates the stamped native ABI; " + "rebuild skippy-server against the current LLAMA_STAGE_BUILD_DIR" + ) + + +def main() -> None: + """Dispatch source snapshots, producer creation, or read-only identity checks.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidate-binary", type=Path) + parser.add_argument("--native-build-dir", type=Path) + producer = parser.add_mutually_exclusive_group() + producer.add_argument("--write-producer", type=Path) + producer.add_argument("--producer-manifest", type=Path) + parser.add_argument("--test-binary", type=Path) + parser.add_argument("--source-snapshot", type=Path) + parser.add_argument("--write-source-snapshot", type=Path) + args = parser.parse_args() + if args.write_source_snapshot: + args.write_source_snapshot.write_text(json.dumps(source_identity()) + "\n", encoding="utf-8") + return + if not args.candidate_binary or not args.native_build_dir: + parser.error("--candidate-binary and --native-build-dir are required") + check_candidate(args.candidate_binary, args.native_build_dir) + if args.write_producer: + if not args.test_binary or not args.source_snapshot: + parser.error("--write-producer requires --test-binary and --source-snapshot") + write_producer(args.write_producer, args.candidate_binary, args.native_build_dir, args.test_binary, args.source_snapshot) + elif args.producer_manifest: + verify_producer(args.producer_manifest, args.candidate_binary, args.native_build_dir) + + +if __name__ == "__main__": + main() diff --git a/scripts/ci-openai-embeddings-smoke.py b/scripts/ci-openai-embeddings-smoke.py new file mode 100755 index 0000000000..f255d78715 --- /dev/null +++ b/scripts/ci-openai-embeddings-smoke.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Official openai-python embeddings smoke against a compatible endpoint.""" + +from __future__ import annotations + +import argparse +import base64 +import math +import struct + +from workload_fixtures import EMBEDDING_INPUTS + + +def main() -> None: + """Check batched numeric and base64 vectors through the official Python SDK.""" + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", required=True) + parser.add_argument("--model", required=True) + args = parser.parse_args() + + try: + from openai import OpenAI + except ModuleNotFoundError as exc: + raise SystemExit( + "openai package not installed; run `python -m pip install openai` first" + ) from exc + + client = OpenAI(api_key="mesh-llm-ci", base_url=args.base_url) + inputs = list(EMBEDDING_INPUTS) + response = client.embeddings.create( + model=args.model, + input=inputs, + encoding_format="float", + ) + if response.object != "list" or response.model != args.model: + raise RuntimeError("embeddings response has the wrong object or model") + if len(response.data) != len(inputs): + raise RuntimeError("embeddings response has the wrong batch size") + dimensions: int | None = None + for index, item in enumerate(response.data): + if item.object != "embedding" or item.index != index: + raise RuntimeError("embeddings response has invalid item metadata") + if not item.embedding or not all(math.isfinite(value) for value in item.embedding): + raise RuntimeError("embeddings response contains no finite vector") + norm = math.sqrt(sum(value * value for value in item.embedding)) + if abs(norm - 1.0) > 1e-4: + raise RuntimeError(f"embedding {index} is not L2-normalized: {norm}") + if dimensions is None: + dimensions = len(item.embedding) + elif dimensions != len(item.embedding): + raise RuntimeError("embedding dimensions differ within one response") + if response.usage.prompt_tokens <= 0: + raise RuntimeError("embeddings usage reports no prompt tokens") + + encoded = client.embeddings.create( + model=args.model, + input=inputs[0], + encoding_format="base64", + ) + if encoded.object != "list" or encoded.model != args.model: + raise RuntimeError("base64 embeddings response has the wrong object or model") + if len(encoded.data) != 1: + raise RuntimeError("base64 embeddings response has the wrong batch size") + item = encoded.data[0] + if item.object != "embedding" or type(item.index) is not int or item.index != 0: + raise RuntimeError("base64 embeddings response has invalid item metadata") + payload = item.embedding + if not isinstance(payload, str): + raise RuntimeError("base64 embedding did not deserialize as a string") + raw = base64.b64decode(payload, validate=True) + if dimensions is None or len(raw) != dimensions * struct.calcsize(" dict: + """POST JSON with a bounded timeout and require an object response.""" + request = urllib.request.Request( + f"{base_url}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=240) as response: + if response.headers.get_content_type() != "application/json": + raise RuntimeError(f"{path} returned non-JSON content") + result = json.load(response) + if not isinstance(result, dict): + raise RuntimeError(f"{path} returned a non-object response") + return result + + +def request_bytes(base_url: str, path: str, payload: dict[str, object]) -> tuple[str, bytes]: + """POST JSON with a bounded timeout and return the response media type and bytes.""" + request = urllib.request.Request( + f"{base_url}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=240) as response: + return response.headers.get_content_type(), response.read() + + +def request_multipart(base_url: str, path: str, model: str, media_path: Path) -> dict: + """Upload a WAV fixture with the selected model and require a JSON object response.""" + boundary = "mesh-llm-workload-smoke" + body = ( + f"--{boundary}\r\n" + 'Content-Disposition: form-data; name="model"\r\n\r\n' + f"{model}\r\n" + f"--{boundary}\r\n" + f'Content-Disposition: form-data; name="file"; filename="{media_path.name}"\r\n' + "Content-Type: audio/wav\r\n\r\n" + ).encode("utf-8") + body += media_path.read_bytes() + f"\r\n--{boundary}--\r\n".encode("ascii") + request = urllib.request.Request( + f"{base_url}{path}", + data=body, + headers={"Content-Type": f"multipart/form-data; boundary={boundary}"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=240) as response: + if response.headers.get_content_type() != "application/json": + raise RuntimeError(f"{path} returned non-JSON content") + result = json.load(response) + if not isinstance(result, dict): + raise RuntimeError(f"{path} returned a non-object response") + return result + + +def smoke_embedding(base_url: str, model: str) -> None: + """Check vector metadata, normalization, semantic ordering, usage, and float/base64 parity.""" + inputs = list(EMBEDDING_INPUTS) + result = request_json( + base_url, + "/embeddings", + {"model": model, "input": inputs, "encoding_format": "float"}, + ) + rows = result.get("data") + if result.get("object") != "list" or result.get("model") != model: + raise RuntimeError("embedding response has the wrong object or model") + if not isinstance(rows, list) or len(rows) != len(inputs): + raise RuntimeError("embedding response has the wrong batch size") + vectors = [] + for index, row in enumerate(rows): + values = row.get("embedding") + if row.get("object") != "embedding" or row.get("index") != index: + raise RuntimeError("embedding response has invalid item metadata") + if not isinstance(values, list) or not values: + raise RuntimeError("embedding response has no vector") + if not all(isinstance(value, (int, float)) and math.isfinite(value) for value in values): + raise RuntimeError("embedding response contains non-finite values") + norm = math.sqrt(sum(value * value for value in values)) + if abs(norm - 1.0) > 1e-4: + raise RuntimeError(f"embedding {index} is not normalized: {norm}") + vectors.append(values) + if len({len(vector) for vector in vectors}) != 1: + raise RuntimeError("embedding dimensions differ within one response") + related = sum(left * right for left, right in zip(vectors[0], vectors[1], strict=True)) + unrelated = sum(left * right for left, right in zip(vectors[0], vectors[2], strict=True)) + if related <= unrelated: + raise RuntimeError( + f"embedding placed unrelated text closer: related={related}, unrelated={unrelated}" + ) + if result.get("usage", {}).get("prompt_tokens", 0) <= 0: + raise RuntimeError("embedding response reported no prompt usage") + encoded = request_json( + base_url, + "/embeddings", + {"model": model, "input": inputs[0], "encoding_format": "base64"}, + ) + if encoded.get("object") != "list" or encoded.get("model") != model: + raise RuntimeError("base64 embedding response has the wrong object or model") + rows = encoded.get("data") + if not isinstance(rows, list) or len(rows) != 1: + raise RuntimeError("base64 embedding response has the wrong batch size") + item = rows[0] + if (not isinstance(item, dict) or item.get("object") != "embedding" + or type(item.get("index")) is not int or item["index"] != 0): + raise RuntimeError("base64 embedding response has invalid item metadata") + payload = item.get("embedding") + if not isinstance(payload, str): + raise RuntimeError("base64 embedding is not a string") + raw = base64.b64decode(payload, validate=True) + if len(raw) != len(vectors[0]) * struct.calcsize(" None: + """Require finite scores with the relevant document strictly ahead of its distractor.""" + result = request_json( + base_url, + "/rerank", + { + "model": model, + "query": RERANK_QUERY, + "documents": list(RERANK_DOCUMENTS), + "return_documents": True, + }, + ) + rows = result.get("results") + if not isinstance(rows, list) or len(rows) != 2: + raise RuntimeError("rerank did not return both documents") + if sorted(row.get("index") for row in rows) != [0, 1]: + raise RuntimeError("rerank returned invalid document indexes") + for row in rows: + score = row.get("relevance_score") + if not isinstance(score, (int, float)) or not math.isfinite(score): + raise RuntimeError("rerank returned a non-finite score") + if not isinstance(row.get("document"), str): + raise RuntimeError("rerank omitted the requested document") + scores = {row["index"]: row["relevance_score"] for row in rows} + if scores[0] <= scores[1]: + raise RuntimeError(f"rerank misplaced the relevant document: {scores}") + if result.get("usage", {}).get("prompt_tokens", 0) <= 0: + raise RuntimeError("rerank did not report prompt usage") + + +def smoke_encoder_decoder(base_url: str, model: str) -> None: + """Verify a deterministic translation contains the expected language anchor.""" + result = request_json( + base_url, + "/completions", + { + "model": model, + "prompt": ENCODER_DECODER_PROMPT, + "max_tokens": 32, + "temperature": 0.0, + }, + ) + choices = result.get("choices") + if not isinstance(choices, list) or len(choices) != 1: + raise RuntimeError("encoder-decoder completion has invalid choices") + if not isinstance(choices[0].get("text"), str) or not choices[0]["text"].strip(): + raise RuntimeError("encoder-decoder completion is empty") + if "haus" not in choices[0]["text"].casefold(): + raise RuntimeError(f"encoder-decoder missed the translation anchor: {choices[0]['text']!r}") + if result.get("usage", {}).get("completion_tokens", 0) <= 0: + raise RuntimeError("encoder-decoder completion reported no generated tokens") + + +def smoke_ocr(base_url: str, model: str, media_path: Path) -> None: + """Submit the image fixture through chat and require a nonempty transcription.""" + image = base64.b64encode(media_path.read_bytes()).decode("ascii") + result = request_json( + base_url, + "/chat/completions", + { + "model": model, + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Read all visible text. Return only the transcription."}, + {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{image}"}}, + ], + } + ], + "max_tokens": 32, + "temperature": 0.0, + }, + ) + choices = result.get("choices") + if not isinstance(choices, list) or len(choices) != 1: + raise RuntimeError("OCR chat completion has invalid choices") + text = choices[0].get("message", {}).get("content") + if not isinstance(text, str) or not text.strip(): + raise RuntimeError("OCR chat completion is empty") + + +def smoke_speech_synthesis(base_url: str, model: str) -> None: + """Validate generated audio framing and nonempty sample content.""" + content_type, audio = request_bytes( + base_url, + "/audio/speech", + { + "model": model, + "input": "The mesh is ready.", + "voice": "default", + "response_format": "wav", + }, + ) + if content_type != "audio/wav": + raise RuntimeError(f"speech synthesis returned {content_type}, not audio/wav") + with wave.open(io.BytesIO(audio)) as sound: + if sound.getnframes() < sound.getframerate() // 10: + raise RuntimeError("speech synthesis returned too little audio") + if sound.getnchannels() < 1 or sound.getsampwidth() != 2: + raise RuntimeError("speech synthesis returned unsupported WAV samples") + samples = sound.readframes(sound.getnframes()) + values = struct.unpack(f"<{len(samples) // 2}h", samples) + rms = math.sqrt(sum(value * value for value in values) / len(values)) + if rms < 1.0: + raise RuntimeError("speech synthesis returned silent audio") + + +def smoke_speech_recognition(base_url: str, model: str, media_path: Path) -> None: + """Submit the audio fixture and require a nonempty transcription.""" + result = request_multipart(base_url, "/audio/transcriptions", model, media_path) + text = result.get("text") + if not isinstance(text, str) or not text.strip(): + raise RuntimeError("speech recognition returned no transcription") + + +def main() -> None: + """Dispatch the workload smoke with its required model and media fixtures.""" + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", required=True) + parser.add_argument("--model", required=True) + parser.add_argument( + "--class", + dest="model_class", + required=True, + choices=("embedding", "rerank", "encoder_decoder", "ocr", "speech_synthesis", "speech_recognition"), + ) + parser.add_argument("--media-path", type=Path) + args = parser.parse_args() + + if args.model_class in {"ocr", "speech_recognition"}: + if args.media_path is None or not args.media_path.is_file(): + parser.error(f"{args.model_class} requires --media-path pointing to a file") + checks = { + "embedding": lambda: smoke_embedding(args.base_url, args.model), + "rerank": lambda: smoke_rerank(args.base_url, args.model), + "encoder_decoder": lambda: smoke_encoder_decoder(args.base_url, args.model), + "ocr": lambda: smoke_ocr(args.base_url, args.model, args.media_path), + "speech_synthesis": lambda: smoke_speech_synthesis(args.base_url, args.model), + "speech_recognition": lambda: smoke_speech_recognition(args.base_url, args.model, args.media_path), + } + checks[args.model_class]() + print(f"OpenAI HTTP {args.model_class} smoke passed: model={args.model}") + + +if __name__ == "__main__": + main() diff --git a/scripts/ci-workload-monolithic-oracle.py b/scripts/ci-workload-monolithic-oracle.py new file mode 100644 index 0000000000..0a6b9c447e --- /dev/null +++ b/scripts/ci-workload-monolithic-oracle.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python3 +"""Compare one Skippy non-chat response with pinned llama.cpp full-model serving. + +Only classes whose OpenAI request and result can be aligned with llama-server +are supported here. This is a numerical/output parity gate, not a general +model-quality benchmark. Both servers must load the same immutable GGUF on CPU. +""" + +from __future__ import annotations + +import argparse +import json +import math +import re +import subprocess +import urllib.request + +from workload_fixtures import ( + EMBEDDING_INPUTS, + ENCODER_DECODER_PROMPT, + RERANK_DOCUMENTS, + RERANK_QUERY, +) + + +EMBEDDING_MAX_ABS_DELTA = 1e-4 +EMBEDDING_MIN_COSINE = 0.99999 +RERANK_MAX_ABS_DELTA = 1e-4 + + +def request_json(base_url: str, path: str, payload: dict[str, object]) -> dict: + """POST JSON with a bounded timeout and require an object response.""" + request = urllib.request.Request( + f"{base_url}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=240) as response: + if response.headers.get_content_type() != "application/json": + raise RuntimeError(f"{base_url}{path} returned non-JSON content") + result = json.load(response) + if not isinstance(result, dict): + raise RuntimeError(f"{base_url}{path} returned a non-object response") + return result + + +def vectors(response: dict, expected_count: int) -> list[list[float]]: + """Validate exact embedding cardinality, indexes, and finite nonempty vectors.""" + rows = response.get("data") + if not isinstance(rows, list) or len(rows) != expected_count: + raise RuntimeError("embedding oracle response has the wrong batch size") + result = [] + for index, row in enumerate(rows): + if not isinstance(row, dict) or row.get("index") != index: + raise RuntimeError("embedding oracle response has invalid indexes") + vector = row.get("embedding") + if not isinstance(vector, list) or not vector: + raise RuntimeError("embedding oracle response has no vector") + if not all(type(value) in (int, float) and math.isfinite(value) for value in vector): + raise RuntimeError("embedding oracle response has non-finite values") + result.append(vector) + return result + + +def compare_embeddings(candidate: dict, reference: dict, expected_count: int = len(EMBEDDING_INPUTS)) -> str: + """Require equal dimensions and bounded numeric disagreement for every input.""" + candidate_vectors = vectors(candidate, expected_count) + reference_vectors = vectors(reference, expected_count) + max_delta = 0.0 + min_cosine = 1.0 + row_metrics = [] + for index, (candidate_vector, reference_vector) in enumerate(zip( + candidate_vectors, reference_vectors, strict=True + )): + if len(candidate_vector) != len(reference_vector): + raise RuntimeError("embedding dimensions differ from monolithic reference") + row_delta = max(abs(left - right) for left, right in zip(candidate_vector, reference_vector)) + max_delta = max(max_delta, row_delta) + dot = sum(left * right for left, right in zip(candidate_vector, reference_vector)) + left_norm = math.sqrt(sum(value * value for value in candidate_vector)) + right_norm = math.sqrt(sum(value * value for value in reference_vector)) + if left_norm == 0 or right_norm == 0: + raise RuntimeError("embedding oracle response has a zero vector") + cosine = dot / (left_norm * right_norm) + min_cosine = min(min_cosine, cosine) + row_metrics.append(f"{index}:delta={row_delta:.7g},cos={cosine:.8g}") + if max_delta > EMBEDDING_MAX_ABS_DELTA or min_cosine < EMBEDDING_MIN_COSINE: + raise RuntimeError( + "embedding differs from monolithic reference: " + f"max_abs_delta={max_delta:.7g}, min_cosine={min_cosine:.8g}, " + f"rows=[{'; '.join(row_metrics)}], " + f"candidate_head={candidate_vectors[0][:5]}, reference_head={reference_vectors[0][:5]}, " + f"candidate_usage={candidate.get('usage')}, reference_usage={reference.get('usage')}" + ) + return f"max_abs_delta={max_delta:.7g}, min_cosine={min_cosine:.8g}" + + +def run_embedding_oracle(candidate_url: str, oracle_url: str, model: str) -> str: + """Compare the shared text fixture in both batched and single-input execution.""" + payload = {"model": model, "input": list(EMBEDDING_INPUTS), "encoding_format": "float"} + candidate = request_json(candidate_url, "/embeddings", payload) + reference = request_json(oracle_url, "/embeddings", payload) + failures = [] + try: + batch_detail = compare_embeddings(candidate, reference) + except RuntimeError as error: + failures.append(f"batched: {error}") + batch_detail = "failed" + single_details = [] + for index, text in enumerate(EMBEDDING_INPUTS): + single_payload = {"model": model, "input": text, "encoding_format": "float"} + candidate = request_json(candidate_url, "/embeddings", single_payload) + reference = request_json(oracle_url, "/embeddings", single_payload) + try: + single_details.append(compare_embeddings(candidate, reference, expected_count=1)) + except RuntimeError as error: + failures.append(f"single[{index}]: {error}") + if failures: + raise RuntimeError("; ".join(failures)) + return f"batch {batch_detail}; singles {', '.join(single_details)}" + + +def indexed_scores(response: dict) -> dict[int, float]: + """Reject missing, duplicate, or invalid rerank document indexes and scores.""" + rows = response.get("results") + if not isinstance(rows, list) or len(rows) != len(RERANK_DOCUMENTS): + raise RuntimeError("rerank oracle response has the wrong document count") + scores = {} + for row in rows: + if not isinstance(row, dict): + raise RuntimeError("rerank oracle response has an invalid row") + index = row.get("index") + score = row.get("relevance_score") + if type(index) is not int or index not in range(len(RERANK_DOCUMENTS)): + raise RuntimeError("rerank oracle response has an invalid document index") + if index in scores or type(score) not in (int, float) or not math.isfinite(score): + raise RuntimeError("rerank oracle response has a duplicate or non-finite score") + scores[index] = float(score) + return scores + + +def compare_rerank(candidate: dict, reference: dict) -> str: + """Require matching relevance order and bounded score differences per document.""" + candidate_scores = indexed_scores(candidate) + reference_scores = indexed_scores(reference) + max_delta = max( + abs(candidate_scores[index] - reference_scores[index]) + for index in range(len(RERANK_DOCUMENTS)) + ) + candidate_order = sorted(candidate_scores, key=lambda index: -candidate_scores[index]) + reference_order = sorted(reference_scores, key=lambda index: -reference_scores[index]) + if candidate_order != reference_order or max_delta > RERANK_MAX_ABS_DELTA: + raise RuntimeError( + "rerank differs from monolithic reference: " + f"candidate_order={candidate_order}, reference_order={reference_order}, " + f"max_abs_delta={max_delta:.7g}" + ) + return f"max_abs_delta={max_delta:.7g}, order={candidate_order}" + + +def completion_text(response: dict) -> str: + """Extract exactly one nonempty completion and normalize only its whitespace.""" + choices = response.get("choices") + if not isinstance(choices, list) or len(choices) != 1: + raise RuntimeError("encoder-decoder oracle response has invalid choices") + text = choices[0].get("text") if isinstance(choices[0], dict) else None + if not isinstance(text, str) or not text.strip(): + raise RuntimeError("encoder-decoder oracle response has no text") + return re.sub(r"\s+", " ", text).strip() + + +def compare_encoder_decoder(candidate: dict, reference: dict) -> str: + """Require exact normalized completion parity with the independent reference.""" + candidate_text = completion_text(candidate) + reference_text = completion_text(reference) + if candidate_text != reference_text: + raise RuntimeError( + "encoder-decoder text differs from monolithic reference: " + f"candidate={candidate_text!r}, reference={reference_text!r}" + ) + return f"identical normalized text={candidate_text!r}" + + +def monolithic_completion(oracle_cli: str, model_path: str) -> dict: + """Run the independent CPU oracle and strip only its terminal end-of-text marker.""" + command = [ + oracle_cli, "-m", model_path, "-p", ENCODER_DECODER_PROMPT, + "-n", "32", "-c", "0", "-b", "2048", "-ub", "2048", "-ngl", "0", + "-s", "1", "--temp", "0", "--no-repack", + "--no-display-prompt", "--simple-io", + ] + try: + result = subprocess.run(command, capture_output=True, text=True, timeout=240, check=False) + except subprocess.TimeoutExpired as error: + raise RuntimeError("monolithic encoder-decoder completion timed out") from error + if result.returncode != 0: + raise RuntimeError( + f"monolithic encoder-decoder completion exited {result.returncode}: " + f"{result.stderr[-1000:]}" + ) + # llama-completion prints this terminal marker after the model emits EOG; + # it is runner metadata, not generated model text. + text = re.sub(r"\s*\[end of text\]\s*$", "", result.stdout).strip() + if not text: + raise RuntimeError("monolithic encoder-decoder completion produced no text") + return {"choices": [{"text": text}]} + + +def main() -> None: + """Select the independent CLI or server oracle for the requested workload.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidate-url", required=True) + parser.add_argument("--oracle-url") + parser.add_argument("--oracle-completion") + parser.add_argument("--model-path") + parser.add_argument("--model", required=True) + parser.add_argument("--class", dest="model_class", required=True, + choices=("embedding", "rerank", "encoder_decoder")) + args = parser.parse_args() + + if args.model_class == "encoder_decoder": + if not args.oracle_completion or not args.model_path or args.oracle_url: + parser.error("encoder_decoder requires --oracle-completion and --model-path only") + payload = {"model": args.model, "prompt": ENCODER_DECODER_PROMPT, + "max_tokens": 32, "temperature": 0.0, "seed": 1} + candidate = request_json(args.candidate_url, "/completions", payload) + reference = monolithic_completion(args.oracle_completion, args.model_path) + detail = compare_encoder_decoder(candidate, reference) + print(f"encoder_decoder local-monolithic oracle passed: {detail}") + return + if not args.oracle_url or args.oracle_completion or args.model_path: + parser.error("embedding/rerank require --oracle-url only") + if args.model_class == "embedding": + detail = run_embedding_oracle(args.candidate_url, args.oracle_url, args.model) + print(f"embedding local-monolithic oracle passed: {detail}") + return + + requests = { + "rerank": ( + "/rerank", + {"model": args.model, "query": RERANK_QUERY, + "documents": list(RERANK_DOCUMENTS), "return_documents": True}, + compare_rerank, + ), + } + path, payload, comparator = requests[args.model_class] + candidate = request_json(args.candidate_url, path, payload) + reference = request_json(args.oracle_url, path, payload) + detail = comparator(candidate, reference) + print(f"{args.model_class} local-monolithic oracle passed: {detail}") + + +if __name__ == "__main__": + main() diff --git a/scripts/generate-ocr-oracle-fixture.py b/scripts/generate-ocr-oracle-fixture.py new file mode 100644 index 0000000000..fb0dc96936 --- /dev/null +++ b/scripts/generate-ocr-oracle-fixture.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Generate an original, deterministic text-bearing PNG for the OCR oracle. + +The 5x7 glyphs below were drawn for this repository. No fonts, image assets, +network services, or third-party packages are required. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path +import struct +import zlib + + +TEXT = "MESH 42" +GLYPHS = { + "M": ("10001", "11011", "10101", "10101", "10001", "10001", "10001"), + "E": ("11111", "10000", "10000", "11110", "10000", "10000", "11111"), + "S": ("01111", "10000", "10000", "01110", "00001", "00001", "11110"), + "H": ("10001", "10001", "10001", "11111", "10001", "10001", "10001"), + "4": ("00110", "01010", "10010", "10010", "11111", "00010", "00010"), + "2": ("11110", "00001", "00001", "01110", "10000", "10000", "11111"), + " ": ("00000",) * 7, +} +SCALE = 12 +MARGIN = 36 +CHAR_SPACING = SCALE +WIDTH = 2 * MARGIN + len(TEXT) * (5 * SCALE + CHAR_SPACING) - CHAR_SPACING +HEIGHT = 2 * MARGIN + 7 * SCALE + + +def chunk(kind: bytes, data: bytes) -> bytes: + """Frame a PNG chunk with a network-order length and CRC.""" + payload = kind + data + return struct.pack(">I", len(data)) + payload + struct.pack(">I", zlib.crc32(payload)) + + +def png_bytes() -> bytes: + """Render the original fixed-glyph OCR fixture as deterministic RGB PNG bytes.""" + pixels = bytearray(b"\xff" * (WIDTH * HEIGHT * 3)) + for char_index, char in enumerate(TEXT): + for glyph_y, row in enumerate(GLYPHS[char]): + for glyph_x, bit in enumerate(row): + if bit == "0": + continue + x0 = MARGIN + char_index * (5 * SCALE + CHAR_SPACING) + glyph_x * SCALE + y0 = MARGIN + glyph_y * SCALE + for y in range(y0, y0 + SCALE): + for x in range(x0, x0 + SCALE): + offset = 3 * (y * WIDTH + x) + pixels[offset : offset + 3] = b"\x00\x00\x00" + rows = b"".join( + b"\x00" + pixels[y * WIDTH * 3 : (y + 1) * WIDTH * 3] + for y in range(HEIGHT) + ) + return ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", struct.pack(">IIBBBBB", WIDTH, HEIGHT, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(rows, level=9)) + + chunk(b"IEND", b"") + ) + + +def main() -> None: + """Write the deterministic original OCR fixture to the requested path.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + args.output.write_bytes(png_bytes()) + print(f"wrote {TEXT!r} OCR oracle fixture: {args.output}") + + +if __name__ == "__main__": + main() diff --git a/scripts/generate-skippy-family-patch.py b/scripts/generate-skippy-family-patch.py index 0af55f4f18..3715d8d56c 100644 --- a/scripts/generate-skippy-family-patch.py +++ b/scripts/generate-skippy-family-patch.py @@ -6,13 +6,12 @@ import argparse import hashlib import json -from pathlib import Path import re import shutil import subprocess import sys import tempfile - +from pathlib import Path PATCH_HEADER = """From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Mesh-LLM CI @@ -165,6 +164,7 @@ def split_model_diff(diff: str) -> dict[str, str]: def load_certified_families(path: Path) -> set[str]: + """Require source mappings for causal split targets, not full-model-only workloads.""" payload = json.loads(path.read_text(encoding="utf-8")) models = payload.get("models") if not isinstance(models, list): @@ -172,7 +172,23 @@ def load_certified_families(path: Path) -> set[str]: families = {model.get("family") for model in models if isinstance(model, dict)} if None in families or len(families) != len(models): raise RuntimeError("family manifest contains missing or duplicate family names") - return families + split_families: set[str] = set() + for model in models: + model_class = model.get("class") + profile = model.get("profile") + if model_class == "causal_generation": + if profile not in ("full", "package-oracle", "graph-only"): + raise RuntimeError("causal family requires a split certification profile") + split_families.add(model["family"]) + elif model_class in ( + "embedding", "rerank", "encoder_decoder", "ocr", + "speech_synthesis", "speech_recognition", + ): + if profile not in ("workload-smoke", "workload-oracle"): + raise RuntimeError("non-chat family requires a workload profile") + else: + raise RuntimeError("family manifest contains a missing or unknown workload class") + return split_families def write_family_shards( diff --git a/scripts/generate-split-certified.py b/scripts/generate-split-certified.py index 4ec7d8431d..a4339353d2 100644 --- a/scripts/generate-split-certified.py +++ b/scripts/generate-split-certified.py @@ -6,12 +6,11 @@ import argparse import hashlib import json -from pathlib import Path import struct import sys +from pathlib import Path from typing import Any - ROOT = Path(__file__).resolve().parents[1] DEFAULT_MANIFEST = ROOT / "ci" / "llama-canary" / "family-certified.json" DEFAULT_OUTPUT = ( @@ -92,6 +91,7 @@ def aggregate_source_sha256(files: list[str], integrity: dict[str, Any]) -> str: def build_roster(manifest: dict[str, Any]) -> dict[str, Any]: + """Bind only causal split-certified artifacts to this exact native recipe.""" policy = manifest.get("policy") models = manifest.get("models") if not isinstance(policy, dict) or not isinstance(models, list): @@ -106,6 +106,18 @@ def build_roster(manifest: dict[str, Any]) -> dict[str, Any]: if not isinstance(model, dict): raise RosterError("family certification model row must be an object") profile_name = model.get("profile") + model_class = model.get("class") + if model_class in ( + "embedding", "rerank", "encoder_decoder", "ocr", + "speech_synthesis", "speech_recognition", + ): + if profile_name not in ("workload-smoke", "workload-oracle"): + raise RosterError("non-chat model cannot claim a split-certified profile") + continue + if model_class != "causal_generation": + raise RosterError("family certification row has a missing or unknown workload class") + if profile_name in ("workload-smoke", "workload-oracle"): + raise RosterError("causal model cannot use a non-chat workload profile") profile = profiles.get(profile_name) if not isinstance(profile, dict) or profile.get("status") != "certified": continue diff --git a/scripts/generate-test-model-manifests.py b/scripts/generate-test-model-manifests.py index 38631d28bb..3a147b0365 100644 --- a/scripts/generate-test-model-manifests.py +++ b/scripts/generate-test-model-manifests.py @@ -13,18 +13,26 @@ import argparse import hashlib import json -from pathlib import Path, PurePosixPath import re import sys +from pathlib import Path, PurePosixPath from typing import Any - ROOT = Path(__file__).resolve().parents[1] REGISTRY = ROOT / "ci" / "model-artifacts" / "registry.json" MANIFEST_DIR = ROOT / "ci" / "model-artifacts" / "manifests" SHA_RE = re.compile(r"^[0-9a-f]{40,64}$") HASH_RE = re.compile(r"^[0-9a-f]{64}$") ID_RE = re.compile(r"^[a-z0-9][a-z0-9._-]*$") +FAMILY_WORKLOAD_CLASSES = { + "causal_generation", + "embedding", + "rerank", + "encoder_decoder", + "ocr", + "speech_synthesis", + "speech_recognition", +} SUITE_OUTPUTS = { "product-smoke": MANIFEST_DIR / "product-smoke.json", @@ -85,7 +93,7 @@ def _artifact(value: Any, field: str) -> dict[str, Any]: revision = _string(artifact.get("revision"), f"{field}.revision") if not SHA_RE.fullmatch(revision): raise RegistryError(f"{field}.revision must be a lowercase immutable SHA") - selector = _string(artifact.get("selector"), f"{field}.selector") + _string(artifact.get("selector"), f"{field}.selector") files = artifact.get("files") if not isinstance(files, list) or not files: raise RegistryError(f"{field}.files must be a non-empty array") @@ -115,6 +123,7 @@ def _artifact(value: Any, field: str) -> dict[str, Any]: def _validate_registry(raw: Any) -> dict[str, Any]: + """Reject ambiguous identities, unauthorized cadences, and invalid certification profiles.""" registry = _object(raw, "registry") _exact_keys( registry, @@ -128,9 +137,9 @@ def _validate_registry(raw: Any) -> dict[str, Any]: policy = _object(registry.get("family_policy"), "registry.family_policy") _exact_keys(policy, {"profiles"}, "registry.family_policy") profiles = _object(policy.get("profiles"), "registry.family_policy.profiles") - expected_profiles = {"full", "package-oracle", "graph-only"} + expected_profiles = {"full", "package-oracle", "graph-only", "workload-smoke", "workload-oracle"} if set(profiles) != expected_profiles: - raise RegistryError("registry.family_policy.profiles must contain the three family profiles") + raise RegistryError("registry.family_policy.profiles must contain the five family profiles") for profile_name, profile in profiles.items(): profile = _object(profile, f"registry.family_policy.profiles.{profile_name}") _exact_keys(profile, {"status", "oracle", "required_lanes"}, f"profile {profile_name}") @@ -173,12 +182,24 @@ def _validate_registry(raw: Any) -> dict[str, Any]: certification = _object(row.get("certification"), f"{field}.certification") _exact_keys( certification, - {"profile", "execution", "resources", "notes", "draft_artifact", "mmproj_artifact"}, + {"class", "profile", "execution", "resources", "notes", "evidence", "draft_artifact", "mmproj_artifact"}, f"{field}.certification", ) + workload_class = _string( + certification.get("class"), f"{field}.certification.class" + ) + if workload_class not in FAMILY_WORKLOAD_CLASSES: + raise RegistryError(f"{field}.certification.class is not a workload class") profile = _string(certification.get("profile"), f"{field}.certification.profile") if profile not in profiles: raise RegistryError(f"{field}.certification.profile is not a family profile") + if profile == "workload-oracle": + evidence = _object(certification.get("evidence"), f"{field}.certification.evidence") + _exact_keys(evidence, {"fixture", "comparison"}, f"{field}.certification.evidence") + _string(evidence.get("fixture"), f"{field}.certification.evidence.fixture") + _string(evidence.get("comparison"), f"{field}.certification.evidence.comparison") + elif "evidence" in certification: + raise RegistryError(f"{field}.certification.evidence requires workload-oracle") _object(certification.get("execution"), f"{field}.certification.execution") _object(certification.get("resources"), f"{field}.certification.resources") _string(certification.get("notes"), f"{field}.certification.notes") @@ -214,6 +235,7 @@ def _family_artifact(artifact: dict[str, Any]) -> dict[str, Any]: def _family_manifest(registry: dict[str, Any]) -> dict[str, Any]: + """Project the full family roster with class-specific execution and evidence policy.""" models: list[dict[str, Any]] = [] for row in registry["artifacts"]: if "llama-family-certification" not in row["suites"]: @@ -221,12 +243,15 @@ def _family_manifest(registry: dict[str, Any]) -> dict[str, Any]: certification = row["certification"] model: dict[str, Any] = { "family": row["family"], + "class": certification["class"], "profile": certification["profile"], "artifact": _family_artifact(row["artifact"]), } for optional in ("draft_artifact", "mmproj_artifact"): if optional in certification: model[optional] = _family_artifact(certification[optional]) + if "evidence" in certification: + model["evidence"] = certification["evidence"] model.update( execution=certification["execution"], resources=certification["resources"], @@ -304,6 +329,7 @@ def compact(item: Any) -> str: [ " {", f' "family": {compact(model["family"])},', + f' "class": {compact(model["class"])},', f' "profile": {compact(model["profile"])},', f' "artifact": {compact(model["artifact"])},', ] @@ -311,6 +337,8 @@ def compact(item: Any) -> str: for optional in ("draft_artifact", "mmproj_artifact"): if optional in model: lines.append(f' {compact(optional)}: {compact(model[optional])},') + if "evidence" in model: + lines.append(f' "evidence": {compact(model["evidence"])},') lines.extend( [ f' "execution": {compact(model["execution"])},', diff --git a/scripts/llama-canary-agent-repair.sh b/scripts/llama-canary-agent-repair.sh index 9a7e16b0ca..db7ae9133f 100755 --- a/scripts/llama-canary-agent-repair.sh +++ b/scripts/llama-canary-agent-repair.sh @@ -392,9 +392,16 @@ run_full_build() { || return 1 run_verification_logged "Skippy smoke tests" "$BUILD_LOG" \ scripts/skippy-ci-smoke.sh || return 1 + run_verification_logged "pinned CPU workload oracles and candidate" "$BUILD_LOG" \ + just skippy-workload-oracles-build "${LLAMA_STAGE_BUILD_DIR:?}-workloads" || return 1 } run_certification() { + local setting + local workload_env=() + while IFS= read -r setting; do + workload_env+=("$setting") + done < <(bash scripts/skippy-workload-oracles-build.sh --print-env "${LLAMA_STAGE_BUILD_DIR:?}-workloads") : > "$CERTIFY_LOG" echo "trusted candidate gate: certify" | tee -a "$CERTIFY_LOG" run_verification_logged "parity manifest validation" "$CERTIFY_LOG" \ @@ -410,6 +417,7 @@ run_certification() { || return 1 run_verification_logged "full supported-family certification" "$CERTIFY_LOG" env \ FAMILY_BATTERY_RUN_ID="$FAMILY_BATTERY_RUN_ID" \ + "${workload_env[@]}" \ scripts/skippy-family-battery.sh --skip-build --plan "$PLAN_PATH" } diff --git a/scripts/llama-oracle-source.py b/scripts/llama-oracle-source.py new file mode 100644 index 0000000000..137de3a762 --- /dev/null +++ b/scripts/llama-oracle-source.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Verify that a test oracle uses the current pinned llama.cpp patch queue.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path +import re +import subprocess +import sys + + +ROOT = Path(__file__).resolve().parents[1] + + +def ordered_patches(patch_dir: Path) -> list[Path]: + """Validate and order both patch series exactly as native source preparation does.""" + patches = sorted(patch_dir.glob("*.patch")) + for expected, patch in enumerate(patches, start=1): + if not re.fullmatch(rf"{expected:04d}-.+\.patch", patch.name): + raise RuntimeError(f"invalid top-level patch sequence: {patch.name}") + generated = patch_dir / "generated" + if not generated.exists(): + return patches + series = generated / "series" + if not series.is_file(): + raise RuntimeError("generated patch directory has no series file") + names = [line.rstrip("\r") for line in series.read_text(encoding="utf-8").splitlines()] + if not names or len(names) != len(list(generated.glob("*.patch"))): + raise RuntimeError("generated patch series does not cover its patch directory") + for expected, name in enumerate(names, start=1): + if not re.fullmatch(rf"{expected:04d}-family-[a-z0-9.-]+(?:--[a-z0-9.-]+)*\.patch", name): + raise RuntimeError(f"invalid generated patch sequence: {name}") + patch = generated / name + if not patch.is_file(): + raise RuntimeError(f"generated patch is missing: {name}") + patches.append(patch) + return patches + + +def patch_digest(patch_dir: Path) -> str: + """Hash patch names and contents in validated application order.""" + digest = hashlib.sha256() + for patch in ordered_patches(patch_dir): + relative_name = patch.relative_to(patch_dir).as_posix() + file_digest = hashlib.sha256(patch.read_bytes()).hexdigest() + digest.update(f"{relative_name}\n{file_digest}\n".encode("utf-8")) + return digest.hexdigest() + + +def prepared_patched_sha(root: Path) -> str: + """Verify the prepared checkout's upstream pin, patch digest, schema, and clean HEAD.""" + checkout = root / ".deps/llama.cpp" + prepared_upstream = (checkout / ".mesh-llm-upstream-sha").read_text(encoding="utf-8").strip() + prepared_patch_digest = (checkout / ".mesh-llm-patch-digest").read_text(encoding="utf-8").strip() + prepared_patched = (checkout / ".mesh-llm-patched-sha").read_text(encoding="utf-8").strip() + prepared_schema = (checkout / ".mesh-llm-prepare-schema").read_text(encoding="utf-8").strip() + upstream = (root / "third_party/llama.cpp/upstream.txt").read_text(encoding="utf-8").strip() + if prepared_schema != "4" or prepared_upstream != upstream: + raise RuntimeError("prepared llama.cpp checkout does not match the pinned upstream") + if prepared_patch_digest != patch_digest(root / "third_party/llama.cpp/patches"): + raise RuntimeError("prepared llama.cpp checkout does not match the current patch queue") + head = subprocess.run( + ["git", "-C", str(checkout), "rev-parse", "HEAD"], + capture_output=True, text=True, check=True, + ).stdout.strip() + if prepared_patched != head: + raise RuntimeError("prepared llama.cpp patched SHA does not match Git HEAD") + subprocess.run(["git", "-C", str(checkout), "diff-index", "--quiet", "HEAD", "--"], + check=True) + return prepared_patched + + +def main() -> None: + """Print the verified patched revision or explain its provenance mismatch.""" + try: + print(prepared_patched_sha(ROOT)) + except (OSError, RuntimeError, subprocess.CalledProcessError) as error: + print(f"pinned llama.cpp oracle source is stale: {error}", file=sys.stderr) + raise SystemExit(1) from error + + +if __name__ == "__main__": + main() diff --git a/scripts/plan-family-battery.py b/scripts/plan-family-battery.py index c6870fced3..e6d23af222 100755 --- a/scripts/plan-family-battery.py +++ b/scripts/plan-family-battery.py @@ -14,17 +14,25 @@ import hashlib import json import os -from pathlib import Path, PurePosixPath import re import struct import sys +from pathlib import Path, PurePosixPath from typing import Any - ROOT = Path(__file__).resolve().parents[1] DEFAULT_MANIFEST = ROOT / "ci" / "llama-canary" / "family-certified.json" CORE_LANES = ("single-step", "chain", "state-handoff") -PROFILE_NAMES = ("full", "package-oracle", "graph-only") +MODEL_CLASS_LANES = { + "causal_generation": CORE_LANES, + "embedding": ("embedding-smoke", "embedding-oracle"), + "rerank": ("rerank-smoke", "rerank-oracle"), + "encoder_decoder": ("encoder-decoder-smoke", "encoder-decoder-oracle"), + "ocr": ("ocr-smoke", "ocr-oracle"), + "speech_synthesis": ("speech-synthesis-smoke", "speech-synthesis-oracle"), + "speech_recognition": ("speech-recognition-smoke", "speech-recognition-oracle"), +} +PROFILE_NAMES = ("full", "package-oracle", "graph-only", "workload-smoke", "workload-oracle") CERTIFIED_PROFILES = ("full", "package-oracle") CERTIFICATION_STATUSES = ("certified", "provisional") ORACLE_KINDS = ("local-monolithic", "independent-trace", "none") @@ -272,12 +280,15 @@ def _load_manifest(path: Path) -> tuple[dict[str, Any], str]: def _validate_policy(value: object) -> dict[str, Any]: + """Enforce each profile's status, oracle type, and exact required lane contract.""" policy = _object(value, "policy") _exact_keys(policy, {"profiles"}, "policy") profiles = _object(policy.get("profiles"), "policy.profiles") if set(profiles) != set(PROFILE_NAMES): - raise PlanError("policy.profiles must define full, package-oracle, and graph-only") + raise PlanError( + "policy.profiles must define full, package-oracle, graph-only, workload-smoke, and workload-oracle" + ) normalized: dict[str, Any] = {} for name in PROFILE_NAMES: profile = _object(profiles[name], f"policy.profiles.{name}") @@ -299,8 +310,19 @@ def _validate_policy(value: object) -> dict[str, Any]: raise PlanError( f"certified profile {name} must require exactly the three core lanes" ) + elif name == "workload-oracle": + if status != "certified" or oracle != "local-monolithic" or tuple(lanes) != ( + "class-specific-smoke", "class-specific-oracle" + ): + raise PlanError("workload-oracle requires certified local-monolithic smoke and oracle lanes") elif status != "provisional" or oracle != "none": - raise PlanError("graph-only must remain provisional and oracle-free") + raise PlanError(f"{name} must remain provisional and oracle-free") + elif name == "graph-only" and tuple(lanes) != ( + "graph-parse", "tensor-ownership", "stage-load" + ): + raise PlanError("graph-only must require exactly the three graph lanes") + elif name == "workload-smoke" and tuple(lanes) != ("class-specific-smoke",): + raise PlanError("workload-smoke must require exactly the class-specific smoke lane") normalized[name] = { "status": status, "oracle": oracle, @@ -314,6 +336,7 @@ def _validate_policy(value: object) -> dict[str, Any]: def _normalize_models(value: object, policy: dict[str, Any]) -> list[dict[str, Any]]: + """Validate family rows and derive their workload-specific lanes and execution budgets.""" if not isinstance(value, list) or not value: raise PlanError("models must be a non-empty array") models: list[dict[str, Any]] = [] @@ -325,6 +348,7 @@ def _normalize_models(value: object, policy: dict[str, Any]) -> list[dict[str, A model, { "family", + "class", "profile", "artifact", "draft_artifact", @@ -332,6 +356,7 @@ def _normalize_models(value: object, policy: dict[str, Any]) -> list[dict[str, A "execution", "resources", "notes", + "evidence", }, field, ) @@ -341,7 +366,28 @@ def _normalize_models(value: object, policy: dict[str, Any]) -> list[dict[str, A if family in seen: raise PlanError(f"duplicate family: {family}") seen.add(family) + model_class = _enum( + model.get("class"), + f"{field}.class", + tuple(MODEL_CLASS_LANES), + ) profile = _enum(model.get("profile"), f"{field}.profile", PROFILE_NAMES) + if model_class != "causal_generation" and profile not in ("workload-smoke", "workload-oracle"): + raise PlanError( + f"{field}.class {model_class} requires a class-specific workload profile" + ) + if model_class == "causal_generation" and profile in ("workload-smoke", "workload-oracle"): + raise PlanError(f"{field}.class causal_generation cannot use workload profiles") + evidence = None + if profile == "workload-oracle": + evidence = _object(model.get("evidence"), f"{field}.evidence") + _exact_keys(evidence, {"fixture", "comparison"}, f"{field}.evidence") + evidence = { + "fixture": _string(evidence.get("fixture"), f"{field}.evidence.fixture"), + "comparison": _string(evidence.get("comparison"), f"{field}.evidence.comparison"), + } + elif "evidence" in model: + raise PlanError(f"{field}.evidence requires workload-oracle") artifact = _artifact(model.get("artifact"), f"{field}.artifact") draft = None if "draft_artifact" in model: @@ -351,6 +397,13 @@ def _normalize_models(value: object, policy: dict[str, Any]) -> list[dict[str, A mmproj = _artifact(model["mmproj_artifact"], f"{field}.mmproj_artifact") if mmproj is not None and len(mmproj["files"]) != 1: raise PlanError(f"{field}.mmproj_artifact.files must name exactly one projector GGUF") + projector_classes = {"ocr", "speech_synthesis", "speech_recognition"} + if model_class in projector_classes and mmproj is None: + raise PlanError(f"{field}.class {model_class} requires an mmproj_artifact") + if model_class != "causal_generation" and len(artifact["files"]) != 1: + raise PlanError( + f"{field}.class {model_class} requires exactly one target GGUF" + ) execution = _object(model.get("execution"), f"{field}.execution") _exact_keys( @@ -380,6 +433,15 @@ def _normalize_models(value: object, policy: dict[str, Any]) -> list[dict[str, A f"{field}.execution.speculative_policy", SPECULATIVE_POLICIES, ) + if model_class != "causal_generation": + if mtp_layers != 0: + raise PlanError( + f"{field}.class {model_class} must not request split or MTP certification" + ) + if speculative_policy != "disabled": + raise PlanError( + f"{field}.class {model_class} must disable speculative decoding" + ) resources = _object(model.get("resources"), f"{field}.resources") _exact_keys( @@ -420,10 +482,15 @@ def _normalize_models(value: object, policy: dict[str, Any]) -> list[dict[str, A models.append( { "family": family, + "class": model_class, "profile": profile, "certification_status": profile_policy["status"], "oracle": profile_policy["oracle"], - "certification_lanes": profile_policy["required_lanes"], + "certification_lanes": ( + profile_policy["required_lanes"] + if model_class == "causal_generation" + else list(MODEL_CLASS_LANES[model_class][:1 if profile == "workload-smoke" else 2]) + ), "artifact": artifact, "draft_artifact": draft, "mmproj_artifact": mmproj, @@ -441,6 +508,7 @@ def _normalize_models(value: object, policy: dict[str, Any]) -> list[dict[str, A "startup_timeout_secs": startup_timeout_secs, }, "notes": notes, + **({"evidence": evidence} if evidence is not None else {}), "manifest_index": index, } ) @@ -573,6 +641,7 @@ def build_plan( shard_count: int = 1, cache_root: Path | None = None, ) -> dict[str, Any]: + """Validate an immutable roster, optionally verify its cache, and shard selected families.""" manifest, manifest_sha256 = _load_manifest(manifest_path) _exact_keys(manifest, {"schema_version", "policy", "models"}, "manifest") policy = _validate_policy(manifest.get("policy")) @@ -603,6 +672,10 @@ def build_plan( "manifest": manifest_source, "manifest_sha256": manifest_sha256, "required_certification_lanes": list(CORE_LANES), + "model_class_lanes": { + model_class: list(lanes) for model_class, lanes in MODEL_CLASS_LANES.items() + }, + "requested_families": families or None, "selected_family_count": len(models), "selected_models": models, "shards": shards, @@ -619,6 +692,7 @@ def _write_github_output(path: Path, plan: dict[str, Any], plan_path: Path) -> N def _parse_args(argv: list[str]) -> argparse.Namespace: + """Parse explicit family selection and read-only cache/plan verification options.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST) parser.add_argument("--families", default="") @@ -627,11 +701,52 @@ def _parse_args(argv: list[str]) -> argparse.Namespace: parser.add_argument("--check-cache", action="store_true") parser.add_argument("--output", type=Path) parser.add_argument("--github-output", type=Path) + parser.add_argument("--verify-plan", type=Path, help="recompute and verify an existing policy plan") + parser.add_argument( + "--inspect-gguf", + type=Path, + help="print the canonical layer count and activation width for one GGUF", + ) return parser.parse_args(argv) def main(argv: list[str] | None = None) -> int: + """Inspect GGUF metadata or generate/verify a source-bound deterministic policy plan.""" args = _parse_args(sys.argv[1:] if argv is None else argv) + if args.inspect_gguf is not None: + dimensions = _gguf_dimensions(args.inspect_gguf) + if dimensions is None: + raise PlanError( + f"GGUF has no positive *.block_count and *.embedding_length metadata: {args.inspect_gguf}" + ) + layer_count, activation_width = dimensions + sys.stdout.write( + json.dumps( + { + "layer_count": layer_count, + "activation_width": activation_width, + }, + sort_keys=True, + ) + + "\n" + ) + return 0 + if args.verify_plan is not None: + supplied = _object(json.loads(args.verify_plan.read_text(encoding="utf-8")), "plan") + requested_families = supplied.get("requested_families") + if requested_families is not None and not isinstance(requested_families, str): + raise PlanError("plan.requested_families must be a string or null") + shards = supplied.get("shards") + if not isinstance(shards, list) or not shards: + raise PlanError("plan.shards must be a nonempty list") + expected = build_plan( + args.manifest, + families=requested_families or "", + shard_count=len(shards), + ) + if supplied != expected: + raise PlanError("policy plan differs from the canonical manifest and selection") + return 0 cache_root = args.cache_root if args.check_cache and cache_root is None: env_cache = os.environ.get("HF_CACHE") or os.environ.get("HF_HOME") diff --git a/scripts/run-command-with-timeout.py b/scripts/run-command-with-timeout.py index ee6a98181c..c7a61d981c 100755 --- a/scripts/run-command-with-timeout.py +++ b/scripts/run-command-with-timeout.py @@ -8,6 +8,7 @@ import signal import subprocess import sys +import time def parse_args() -> argparse.Namespace: @@ -41,39 +42,53 @@ def terminate_group(process: subprocess.Popen[bytes]) -> None: def main() -> int: args = parse_args() - # The wrapper is commonly invoked from manifest-reading shell loops. A - # child must never inherit and consume the loop's stdin, because doing so - # can silently drop later planned rows. Commands in this harness are fully - # argument-driven, so EOF is the only valid stdin contract. - process = subprocess.Popen( - args.command, - stdin=subprocess.DEVNULL, - start_new_session=True, - ) + received_signal: int | None = None - def terminate_on_signal(signum: int, _frame: object) -> None: - # Prevent a second cancellation signal from interrupting cleanup and - # leaving descendants behind on the persistent runner. - signal.signal(signal.SIGINT, signal.SIG_IGN) - signal.signal(signal.SIGTERM, signal.SIG_IGN) - print( - f"{args.label} received signal {signum}; terminating process group", - file=sys.stderr, - ) - terminate_group(process) - raise SystemExit(128 + signum) + def request_termination(signum: int, _frame: object) -> None: + # A handler can interrupt Popen construction or wait's internal lock. + # Only record intent here; never wait, print, or clean up reentrantly. + nonlocal received_signal + if received_signal is None: + received_signal = signum - signal.signal(signal.SIGINT, terminate_on_signal) - signal.signal(signal.SIGTERM, terminate_on_signal) + previous_handlers = { + signum: signal.signal(signum, request_termination) + for signum in (signal.SIGINT, signal.SIGTERM) + } try: - return process.wait(timeout=args.seconds) - except subprocess.TimeoutExpired: + # Commands are argument-driven: inheriting a manifest loop's stdin + # could silently consume later planned rows, so children receive EOF. + process = subprocess.Popen( + args.command, + stdin=subprocess.DEVNULL, + start_new_session=True, + ) + deadline = time.monotonic() + args.seconds + while received_signal is None: + remaining = deadline - time.monotonic() + if remaining <= 0: + print( + f"{args.label} timed out after {args.seconds}s; terminating process group", + file=sys.stderr, + ) + terminate_group(process) + return 124 + try: + returncode = process.wait(timeout=min(0.1, remaining)) + except subprocess.TimeoutExpired: + continue + if received_signal is None: + return returncode + print( - f"{args.label} timed out after {args.seconds}s; terminating process group", + f"{args.label} received signal {received_signal}; terminating process group", file=sys.stderr, ) terminate_group(process) - return 124 + return 128 + received_signal + finally: + for signum, handler in previous_handlers.items(): + signal.signal(signum, handler) if __name__ == "__main__": diff --git a/scripts/skippy-family-battery.sh b/scripts/skippy-family-battery.sh index 2795c6fa90..ec8e9ddd42 100755 --- a/scripts/skippy-family-battery.sh +++ b/scripts/skippy-family-battery.sh @@ -3,11 +3,11 @@ set -euo pipefail # Supported-families certification battery (issue #1434; tiers dropped 2026-08-25). # -# Every row of the single manifest gets one consolidated core certification: -# single-step, chain, and state-handoff lanes. Models with MTP/NextN tensors -# require the native draft sideband and verify it against the target. The -# production topology planner exhaustively classifies boundaries and selects -# accepted balanced cuts before any model is loaded. +# Causal-generation rows get single-step, chain, and state-handoff parity. +# The production topology planner classifies all boundaries and selects +# accepted balanced cuts. MTP/NextN sidebands are verified against the target. +# Non-chat rows instead run class-specific smoke and independent monolithic +# oracle lanes; these full-model-only classes never enter split certification. # # Models are NEVER cached through GitHub Actions cache. The family-certify # runner ships a large pre-warmed, read-only HF cache. When HF_CACHE is set, @@ -119,9 +119,11 @@ require_cmd() { } } -require_cmd hf require_cmd jq require_cmd python3 +if (( DRY_RUN == 0 )); then + require_cmd hf +fi if [[ ! -x "$PLANNER" ]]; then echo "family battery planner is not executable: $PLANNER" >&2 exit 1 @@ -154,9 +156,12 @@ if [[ -n "$FAMILY_FILTER" && ! "$FAMILY_FILTER" =~ ^[a-zA-Z0-9._-]+(,[a-zA-Z0-9. fi mkdir -p "$MODEL_SCAN_DIR" "$PREFLIGHT_DIR" "$TOPOLOGY_PLAN_DIR" "$CERT_DIR" +if [[ -n "${SKIPPY_WORKLOAD_PRODUCER_MANIFEST:-}" ]]; then + cp "$SKIPPY_WORKLOAD_PRODUCER_MANIFEST" "$ARTIFACT_DIR/workload-producer.json" +fi : > "$RESULTS_JSONL" printf 'family\tmodel_id\tsource_revision\tmodel_path\tmtp_layers\n' > "$NATIVE_MTP_MODELS_TSV" -printf 'family|repo|source_revision|file|selector|two_stage_split|three_stage_splits|topology_plan|layer_end|notes|target_path|native_mtp|model_size_bytes|mtp_layers|activation_width|startup_timeout_secs|mmproj_repo|mmproj_revision|mmproj_file|mmproj_path\n' > "$RESOLVED_MANIFEST" +printf 'family|class|repo|source_revision|file|selector|two_stage_split|three_stage_splits|topology_plan|layer_end|notes|target_path|native_mtp|model_size_bytes|mtp_layers|activation_width|startup_timeout_secs|lane_csv|mmproj_repo|mmproj_revision|mmproj_file|mmproj_path\n' > "$RESOLVED_MANIFEST" prepare_policy_plan() { local plan_args=( @@ -185,6 +190,7 @@ prepare_policy_plan() { "${plan_args[@]}" fi + "$PLANNER" --manifest "$MANIFEST" --verify-plan "$POLICY_PLAN_COPY" python3 - "$MANIFEST" "$POLICY_PLAN_COPY" "$SHARD_INDEX" <<'PY' import hashlib import json @@ -195,12 +201,23 @@ manifest_path, plan_path, shard_index = sys.argv[1:] manifest_sha = hashlib.sha256(Path(manifest_path).read_bytes()).hexdigest() plan = json.loads(Path(plan_path).read_text(encoding="utf-8")) core = ["single-step", "chain", "state-handoff"] +class_lanes = { + "causal_generation": core, + "embedding": ["embedding-smoke", "embedding-oracle"], + "rerank": ["rerank-smoke", "rerank-oracle"], + "encoder_decoder": ["encoder-decoder-smoke", "encoder-decoder-oracle"], + "ocr": ["ocr-smoke", "ocr-oracle"], + "speech_synthesis": ["speech-synthesis-smoke", "speech-synthesis-oracle"], + "speech_recognition": ["speech-recognition-smoke", "speech-recognition-oracle"], +} if plan.get("schema_version") != 1: raise SystemExit("policy plan has an unsupported schema_version") if plan.get("manifest_sha256") != manifest_sha: raise SystemExit("policy plan does not match the checked-in manifest bytes") if plan.get("required_certification_lanes") != core: raise SystemExit("policy plan does not preserve the three-lane certification contract") +if plan.get("model_class_lanes") != class_lanes: + raise SystemExit("policy plan does not preserve the model-class lane contract") if not plan.get("selected_models"): raise SystemExit("policy plan selected no models") if shard_index: @@ -355,6 +372,7 @@ scan_model() { MODEL_SIZE_BYTES=0 MODEL_MTP_LAYERS="" MODEL_LAYER_COUNT=0 + MODEL_ACTIVATION_WIDTH=0 if (( DRY_RUN == 1 )); then echo "$BIN_DIR/skippy-model-package inspect '$target' > '$scan_json'" @@ -373,7 +391,19 @@ scan_model() { fi MODEL_SIZE_BYTES="$(jq '[.tensors[].byte_size] | add // 0' "$scan_json")" - MODEL_LAYER_COUNT="$(jq '[.tensors[] | select(.layer_index != null) | .layer_index] | unique | length' "$scan_json")" + local dimensions + if ! dimensions="$("$PLANNER" --inspect-gguf "$target")"; then + jq -n \ + --arg family "$family" \ + --arg model_id "$model_id" \ + --arg target "$target" \ + '{family:$family,model_id:$model_id,target_model:$target,exit_code:1,outcomes:[{name:"model-metadata",status:"fail",outcome:"model-invalid",note:"could not read canonical GGUF dimensions"}]}' \ + >> "$RESULTS_JSONL" + FAILURES+=("$family(metadata)") + return 1 + fi + MODEL_LAYER_COUNT="$(jq -r '.layer_count' <<<"$dimensions")" + MODEL_ACTIVATION_WIDTH="$(jq -r '.activation_width' <<<"$dimensions")" MODEL_MTP_LAYERS="$(jq -r ' [.tensors[] | select(.layer_index != null) @@ -397,14 +427,22 @@ scan_model() { preflight_environment() { local model_root="${HF_HOME:-$(dirname "$PREFLIGHT_FIRST_TARGET")}" - python3 - "$ARTIFACT_DIR" "$model_root" "$MIN_FREE_GIB" "$PREFLIGHT_DIR/environment.json" <<'PY' + local port_mode="full" needs_oracle_server=0 + if ! jq -e '[.selected_models[].class] | any(. == "causal_generation")' "$POLICY_PLAN_COPY" >/dev/null; then + port_mode="workload" + if jq -e '[.selected_models[].class] | any(. == "embedding" or . == "rerank" or . == "ocr" or . == "speech_recognition")' "$POLICY_PLAN_COPY" >/dev/null; then + needs_oracle_server=1 + fi + fi + python3 - "$ARTIFACT_DIR" "$model_root" "$MIN_FREE_GIB" "$PREFLIGHT_ONLY" "$PREFLIGHT_DIR/environment.json" \ + "$port_mode" "${SKIPPY_WORKLOAD_OPENAI_PORT:-19337}" "${SKIPPY_WORKLOAD_ORACLE_PORT:-19338}" "$needs_oracle_server" <<'PY' import json import shutil import socket import sys from pathlib import Path -artifact_root, model_root, minimum_gib, output = sys.argv[1:] +artifact_root, model_root, minimum_gib, preflight_only, output, port_mode, candidate_port, oracle_port, needs_oracle_server = sys.argv[1:] minimum_bytes = int(minimum_gib) * 1024**3 filesystems = [] for label, path_text in (("artifacts", artifact_root), ("models", model_root)): @@ -423,20 +461,32 @@ for label, path_text in (("artifacts", artifact_root), ("models", model_root)): ) busy_ports = [] -for port in range(19000, 20032): - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - try: - sock.settimeout(0.01) - if sock.connect_ex(("127.0.0.1", port)) == 0: +ports = list(range(19000, 20032)) if port_mode == "full" else [int(candidate_port)] +if port_mode == "workload" and needs_oracle_server == "1": + ports.append(int(oracle_port)) +if any(port < 1 or port > 65535 for port in ports) or len(ports) != len(set(ports)): + raise SystemExit("invalid or conflicting workload certification ports") +if preflight_only == "0": + for port in ports: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + sock.settimeout(0.01) + if sock.connect_ex(("127.0.0.1", port)) == 0: + busy_ports.append(port) + except OSError: busy_ports.append(port) - except OSError: - busy_ports.append(port) - finally: - sock.close() + finally: + sock.close() report = { "filesystems": filesystems, - "port_range": {"start": 19000, "end": 20031, "busy": busy_ports}, + "port_range": { + "start": min(ports), + "end": max(ports), + "ports_checked": ports if port_mode == "workload" else None, + "checked": preflight_only == "0", + "busy": busy_ports, + }, } Path(output).write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") if any(not item["sufficient"] for item in filesystems) or busy_ports: @@ -568,13 +618,17 @@ preflight_manifest() { return 1 fi - while IFS='|' read -r family profile repo source_revision file selector layer_end activation_width notes expected_model_bytes startup_timeout_override expected_mtp_layers lane_csv _speculative_policy mmproj_repo mmproj_revision mmproj_file; do - if [[ "$profile" != "full" ]]; then - echo "the local monolithic battery cannot execute profile $profile for $family" >&2 + while IFS='|' read -r family model_class profile repo source_revision file selector layer_end activation_width notes expected_model_bytes startup_timeout_override expected_mtp_layers lane_csv _speculative_policy mmproj_repo mmproj_revision mmproj_file; do + if [[ "$model_class" == "causal_generation" && "$profile" != "full" ]] || + [[ "$model_class" != "causal_generation" && "$profile" != "workload-smoke" && "$profile" != "workload-oracle" ]]; then + echo "the family battery cannot execute profile $profile for $model_class family $family" >&2 exit 1 fi - if [[ "$lane_csv" != "single-step,chain,state-handoff" ]]; then - echo "certified family $family does not preserve the three-lane contract" >&2 + local expected_lane_csv + expected_lane_csv="$(jq -r --arg model_class "$model_class" --arg profile "$profile" \ + '.model_class_lanes[$model_class] | if $profile == "workload-smoke" then .[:1] else . end | join(",")' "$plan")" + if [[ -z "$expected_lane_csv" || "$lane_csv" != "$expected_lane_csv" ]]; then + echo "family $family does not preserve the $model_class lane contract" >&2 exit 1 fi @@ -613,6 +667,13 @@ preflight_manifest() { record_preflight_outcome "model-preflight" "$family" "$model_id" "fail" "model-invalid" "planned runtime range $layer_end does not match scanned layer count $MODEL_LAYER_COUNT" continue fi + if (( MODEL_ACTIVATION_WIDTH != activation_width )); then + echo "policy/runtime activation-width mismatch for $family: planned $activation_width, scanned $MODEL_ACTIVATION_WIDTH" >&2 + FAILURES+=("$family(activation-width)") + PREFLIGHT_FAILURE_COUNT=$((PREFLIGHT_FAILURE_COUNT + 1)) + record_preflight_outcome "model-preflight" "$family" "$model_id" "fail" "model-invalid" "planned activation width $activation_width does not match GGUF metadata $MODEL_ACTIVATION_WIDTH" + continue + fi if (( actual_mtp_layers != expected_mtp_layers )); then echo "policy/MTP mismatch for $family: planned $expected_mtp_layers, scanned $actual_mtp_layers" >&2 FAILURES+=("$family(mtp-policy)") @@ -632,7 +693,11 @@ preflight_manifest() { local family_slug topology_plan two_stage_split three_stage_splits family_slug="$(slugify "$family")" topology_plan="$TOPOLOGY_PLAN_DIR/$family_slug.json" - if (( DRY_RUN == 0 )); then + if [[ "$model_class" != "causal_generation" ]]; then + topology_plan="" + two_stage_split="" + three_stage_splits="" + elif (( DRY_RUN == 0 )); then if ! "$BIN_DIR/skippy-topology-plan" "$model_id" "$layer_end" "$activation_width" >"$topology_plan"; then echo "product topology planning failed for $family" >&2 FAILURES+=("$family(topology-plan)") @@ -681,9 +746,9 @@ preflight_manifest() { mmproj_path="/$mmproj_repo/$mmproj_file" fi fi - printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \ - "$family" "$repo" "$source_revision" "$file" "$selector" "$two_stage_split" "$three_stage_splits" "$topology_plan" "$layer_end" "$notes" "$target" \ - "$MODEL_HAS_MTP" "$MODEL_SIZE_BYTES" "$MODEL_MTP_LAYERS" "$activation_width" "$startup_timeout" \ + printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \ + "$family" "$model_class" "$repo" "$source_revision" "$file" "$selector" "$two_stage_split" "$three_stage_splits" "$topology_plan" "$layer_end" "$notes" "$target" \ + "$MODEL_HAS_MTP" "$MODEL_SIZE_BYTES" "$MODEL_MTP_LAYERS" "$activation_width" "$startup_timeout" "$lane_csv" \ "$mmproj_repo" "$mmproj_revision" "$mmproj_file" "$mmproj_path" \ >> "$RESOLVED_MANIFEST" if (( DRY_RUN == 0 )); then @@ -702,6 +767,7 @@ preflight_manifest() { | select(.family as $family | $selected | index($family)) | [ .family, + .class, .profile, .artifact.repo, .artifact.revision, @@ -751,11 +817,19 @@ preflight_manifest() { return 1 fi if ! preflight_environment; then - record_preflight_outcome "environment-preflight" "battery" "environment" "fail" "harness" "insufficient disk headroom or occupied certification ports; see preflight/environment.json" + local environment_failure_note="insufficient disk headroom; see preflight/environment.json" + if (( PREFLIGHT_ONLY == 0 )); then + environment_failure_note="insufficient disk headroom or occupied certification ports; see preflight/environment.json" + fi + record_preflight_outcome "environment-preflight" "battery" "environment" "fail" "harness" "$environment_failure_note" PREFLIGHT_FAILURE_COUNT=$((PREFLIGHT_FAILURE_COUNT + 1)) return 1 fi - record_preflight_outcome "environment-preflight" "battery" "environment" "pass" "pass" "disk headroom and certification port range validated" + local environment_note="disk headroom validated; certification ports not checked in preflight-only mode" + if (( PREFLIGHT_ONLY == 0 )); then + environment_note="disk headroom and certification port range validated" + fi + record_preflight_outcome "environment-preflight" "battery" "environment" "pass" "pass" "$environment_note" } @@ -812,11 +886,119 @@ run_mmproj_smoke() { >> "$RESULTS_JSONL" } +run_workload_certify() { + local family="$1" model_class="$2" target="$3" model_id="$4" source_revision="$5" + local startup_timeout="$6" model_size_bytes="$7" lane_csv="$8" mmproj="$9" + local cert_run_id cert_run_dir exit_code log_path cert_timeout oracle_requested + local smoke_lane oracle_lane certified oracle_executable + TOTAL=$((TOTAL + 1)) + cert_timeout="$(cert_timeout_for_startup "$startup_timeout")" + cert_run_id="$(printf '%03d-%s-%s' "$TOTAL" "$(slugify "$family")" "$(slugify "$model_class")")" + cert_run_dir="$CERT_DIR/$cert_run_id" + mkdir -p "$cert_run_dir" + log_path="$cert_run_dir/workload-certification.log" + smoke_lane="${lane_csv%%,*}" + oracle_lane="" + certified=0 + if [[ "$lane_csv" == *,* ]]; then + oracle_lane="${lane_csv#*,}" + certified=1 + fi + local command=( + "$ROOT/scripts/skippy-workload-certify.sh" + --class "$model_class" + --lane "$smoke_lane" + --model-path "$target" + --model-id "$model_id" + --work-dir "$cert_run_dir" + --startup-timeout-secs "$startup_timeout" + --skip-build + ) + if [[ -n "$mmproj" ]]; then + command+=(--projector-path "$mmproj") + fi + oracle_requested=0 + oracle_executable="" + if [[ -n "${SKIPPY_WORKLOAD_ORACLE_SERVER:-}" ]] && + [[ "$model_class" =~ ^(embedding|rerank|ocr|speech_recognition)$ ]]; then + command+=(--oracle-server "$SKIPPY_WORKLOAD_ORACLE_SERVER") + oracle_requested=1 + oracle_executable="$SKIPPY_WORKLOAD_ORACLE_SERVER" + fi + if [[ -n "${SKIPPY_WORKLOAD_ORACLE_COMPLETION:-}" ]] && + [[ "$model_class" == "encoder_decoder" ]]; then + command+=(--oracle-completion "$SKIPPY_WORKLOAD_ORACLE_COMPLETION") + oracle_requested=1 + oracle_executable="$SKIPPY_WORKLOAD_ORACLE_COMPLETION" + fi + if [[ -n "${SKIPPY_WORKLOAD_ORACLE_TTS:-}" ]] && + [[ "$model_class" == "speech_synthesis" ]]; then + command+=(--oracle-tts "$SKIPPY_WORKLOAD_ORACLE_TTS") + oracle_requested=1 + oracle_executable="$SKIPPY_WORKLOAD_ORACLE_TTS" + fi + if (( certified == 1 )); then + command+=(--require-oracle) + fi + echo "==> workload certification: family=$family class=$model_class lanes=$lane_csv model=$(basename "$target")" + if (( DRY_RUN == 1 )); then + printf '%q ' "${command[@]}" + printf '\n' + return 0 + fi + exit_code=0 + if (( certified == 1 && oracle_requested != 1 )); then + echo "certified workload $family ($model_class) requires a class-appropriate local-monolithic oracle executable" | tee "$log_path" >&2 + exit_code=1 + else + "$ROOT/scripts/run-command-with-timeout.py" \ + --seconds "$cert_timeout" \ + --label "workload certification $family ($model_class)" \ + -- "${command[@]}" >"$log_path" 2>&1 || exit_code=$? + fi + if (( certified == 1 && exit_code == 0 )); then + local verify_command=(python3 "$ROOT/scripts/verify-workload-oracle-evidence.py" \ + --evidence "$cert_run_dir/workload-oracle-evidence.json" \ + --class "$model_class" --smoke-lane "$smoke_lane" --oracle-lane "$oracle_lane" \ + --model-id "$model_id" --model-path "$target" \ + --candidate-executable "${SKIPPY_WORKLOAD_CANDIDATE_BIN_DIR:-$ROOT/target/debug}/skippy-server" \ + --oracle-executable "$oracle_executable" \ + --pinned-patch-sha "$(python3 "$ROOT/scripts/llama-oracle-source.py")") + if [[ -n "$mmproj" ]]; then + verify_command+=(--projector-path "$mmproj") + fi + "${verify_command[@]}" >>"$log_path" 2>&1 || exit_code=$? + fi + jq -n \ + --arg family "$family" \ + --arg model_id "$model_id" \ + --arg source_revision "$source_revision" \ + --arg workload_class "$model_class" \ + --arg smoke_lane "$smoke_lane" \ + --arg oracle_lane "$oracle_lane" \ + --arg log "$log_path" \ + --argjson model_size_bytes "$model_size_bytes" \ + --argjson startup_timeout_secs "$startup_timeout" \ + --argjson certification_timeout_secs "$cert_timeout" \ + --argjson exit_code "$exit_code" \ + --argjson oracle_requested "$oracle_requested" \ + '{family:$family,model_id:$model_id,source_revision:$source_revision,workload_class:$workload_class,certification_status:(if $oracle_lane != "" then "certified" else "provisional" end),oracle:(if $oracle_lane != "" and $exit_code == 0 then "local-monolithic" else "none" end),model_size_bytes:$model_size_bytes,startup_timeout_secs:$startup_timeout_secs,certification_timeout_secs:$certification_timeout_secs,exit_code:$exit_code,outcomes:([{name:$smoke_lane,status:(if $exit_code == 0 then "pass" else "fail" end),outcome:(if $exit_code == 0 then "smoke-pass" elif $exit_code == 124 then "timeout" else "harness" end),exit_code:$exit_code,log:$log}] + (if $oracle_lane == "" then [] else [{name:$oracle_lane,status:(if $exit_code == 0 then "pass" else "fail" end),outcome:(if $exit_code == 0 then "oracle-pass" elif $exit_code == 124 then "timeout" else "harness" end),exit_code:$exit_code,log:$log}] end))}' \ + >> "$RESULTS_JSONL" + if (( exit_code != 0 )); then + FAILURES+=("$family@$model_class") + CERT_FAILURE_COUNT=$((CERT_FAILURE_COUNT + 1)) + fi +} + run_resolved_manifest() { local resolved_manifest="$1" - while IFS='|' read -r family repo source_revision file selector split_layer chain_splits _topology_plan layer_end _notes target native_mtp model_size_bytes _mtp_layers activation_width startup_timeout mmproj_repo mmproj_revision mmproj_file mmproj_path; do + while IFS='|' read -r family model_class repo source_revision file selector split_layer chain_splits _topology_plan layer_end _notes target native_mtp model_size_bytes _mtp_layers activation_width startup_timeout lane_csv mmproj_repo mmproj_revision mmproj_file mmproj_path; do [[ "$family" == "family" ]] && continue local model_id="$repo:$selector" + if [[ "$model_class" != "causal_generation" ]]; then + run_workload_certify "$family" "$model_class" "$target" "$model_id" "$source_revision" "$startup_timeout" "$model_size_bytes" "$lane_csv" "$mmproj_path" + continue + fi run_certify "$family" "$target" "$model_id" "$source_revision" "$split_layer" "$chain_splits" "$layer_end" "$native_mtp" "$startup_timeout" "$model_size_bytes" "$activation_width" if [[ -n "$mmproj_repo" ]]; then @@ -825,12 +1007,23 @@ run_resolved_manifest() { done < "$resolved_manifest" } +write_lane_summary() { + jq -sr --slurpfile policy "$POLICY_PLAN_COPY" ' + def model_class($row): + $row.workload_class // + ([$policy[0].selected_models[] | select(.family == $row.family) | .class][0] // ""); + ["family","class","split_layer","lane","status","outcome","exit_code"], + (.[] as $row | $row.outcomes[] | [$row.family,model_class($row),($row.split_layer // ""),.name,.status,.outcome,.exit_code]) + | @tsv + ' "$RESULTS_JSONL" > "$SUMMARY_TSV" +} + build_certification_binaries if ! preflight_manifest "$POLICY_PLAN_COPY"; then echo "family battery preflight failed; no certification lane was started" >&2 elif (( PREFLIGHT_ONLY == 0 )); then EXPECTED_TOTAL="$(planned_certification_count "$RESOLVED_MANIFEST")" - EXPECTED_MM_SMOKE_TOTAL="$(tail -n +2 "$RESOLVED_MANIFEST" | awk -F'|' '$17 != "" { count += 1 } END { print count + 0 }')" + EXPECTED_MM_SMOKE_TOTAL="$(tail -n +2 "$RESOLVED_MANIFEST" | awk -F'|' '$2 == "causal_generation" && $19 != "" { count += 1 } END { print count + 0 }')" run_resolved_manifest "$RESOLVED_MANIFEST" if (( TOTAL != EXPECTED_TOTAL )); then echo "executed $TOTAL certifications but validated plan requires $EXPECTED_TOTAL" >&2 @@ -843,7 +1036,7 @@ elif (( PREFLIGHT_ONLY == 0 )); then CERT_FAILURE_COUNT=$((CERT_FAILURE_COUNT + 1)) fi if (( DRY_RUN == 0 )); then - actual_result_count="$(jq -s '[.[] | select(.split_layer != null)] | length' "$RESULTS_JSONL")" + actual_result_count="$(jq -s '[.[] | select(.split_layer != null or .workload_class != null)] | length' "$RESULTS_JSONL")" if (( actual_result_count != EXPECTED_TOTAL )); then echo "recorded $actual_result_count certification results but validated plan requires $EXPECTED_TOTAL" >&2 FAILURES+=("battery(result-reconciliation)") @@ -854,11 +1047,7 @@ fi echo if (( DRY_RUN == 0 )); then - jq -sr ' - ["family","split_layer","lane","status","outcome","exit_code"], - (.[] as $row | $row.outcomes[] | [$row.family,($row.split_layer // ""),.name,.status,.outcome,.exit_code]) - | @tsv - ' "$RESULTS_JSONL" > "$SUMMARY_TSV" + write_lane_summary { echo "# Supported-families battery" echo @@ -869,6 +1058,9 @@ if (( DRY_RUN == 0 )); then fi echo "- Certifications: $TOTAL" echo "- Planned certifications: $EXPECTED_TOTAL" + echo "- Certified-profile families: $(jq '[.selected_models[] | select(.certification_status == "certified")] | length' "$POLICY_PLAN_COPY")" + echo "- Provisional workload-smoke families: $(jq '[.selected_models[] | select(.profile == "workload-smoke")] | length' "$POLICY_PLAN_COPY")" + echo "- Certified non-chat workloads require separate smoke and explicit local-monolithic oracle evidence." echo "- Multimodal smokes: $MM_SMOKE_TOTAL (planned: $EXPECTED_MM_SMOKE_TOTAL; failures: $MM_SMOKE_FAILURE_COUNT)" echo "- Native MTP models: $(( $(wc -l < "$NATIVE_MTP_MODELS_TSV") - 1 ))" echo "- Preflight failures: $PREFLIGHT_FAILURE_COUNT" @@ -894,6 +1086,8 @@ fi if (( PREFLIGHT_ONLY == 1 )); then echo "family battery preflight complete: $PREFLIGHT_FAILURE_COUNT failures" +elif (( DRY_RUN == 1 )); then + echo "family battery dry run complete: $TOTAL certifications planned; no lanes executed" else echo "family battery complete: $((TOTAL - CERT_FAILURE_COUNT))/$TOTAL certifications passed" fi diff --git a/scripts/skippy-ocr-asr-oracle.py b/scripts/skippy-ocr-asr-oracle.py new file mode 100644 index 0000000000..b9e8f81729 --- /dev/null +++ b/scripts/skippy-ocr-asr-oracle.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +"""Compare OCR or ASR output with pinned llama.cpp monolithic serving. + +Both HTTP servers must load the same GGUF and projector on CPU. Matching text +is a local execution-parity check, not general model-quality certification. +OCR additionally checks the known text in the generated fixture. ASR requires +an independently labeled audio fixture before semantic accuracy can be claimed. +""" + +from __future__ import annotations + +import argparse +import base64 +import json +from pathlib import Path +import re +import unicodedata +import urllib.request + + +OCR_PROMPT = "Read all visible text. Return only the transcription." +OCR_FIXTURE_TEXT = "MESH 42" +ASR_PROMPT = "Transcribe audio to text" +ASR_MAX_TOKENS = 128 +BOUNDARY = "mesh-llm-ocr-asr-oracle" +NON_TRANSCRIPT_PREFIXES = ( + "i can t fulfill", + "i cannot fulfill", + "i can help you with transcribing", + "you can use various tools to transcribe", +) + + +def normalized_text(value: object, source: str) -> str: + """Normalize Unicode, case, and punctuation while rejecting empty output.""" + if not isinstance(value, str): + raise RuntimeError(f"{source} returned no text") + normalized = unicodedata.normalize("NFKC", value).casefold() + normalized = re.sub(r"[^\w]+", " ", normalized, flags=re.UNICODE).strip() + if not normalized: + raise RuntimeError(f"{source} returned empty text") + return normalized + + +def transcription_text(value: object, source: str) -> str: + """Remove only known presentation labels and reject refusals masquerading as ASR.""" + text = normalized_text(value, source) + # The two frontends add different presentational labels around the same + # transcript. Strip only these exact known prefixes, never content words. + for prefix in ("the text is ", "the audio is "): + if text.startswith(prefix): + text = text[len(prefix):] + break + if text.startswith(NON_TRANSCRIPT_PREFIXES): + raise RuntimeError(f"{source} returned a refusal or generic ASR advice, not a transcript") + return text + + +def compare_text(candidate: object, reference: object, expected: str | None, + *, transcript: bool = False) -> str: + """Require exact normalized parity and, when provided, the independent fixture label.""" + normalizer = transcription_text if transcript else normalized_text + candidate_text = normalizer(candidate, "candidate") + reference_text = normalizer(reference, "monolithic reference") + if candidate_text != reference_text: + raise RuntimeError( + "text differs from monolithic reference: " + f"candidate={candidate_text!r}, reference={reference_text!r}" + ) + if expected is not None: + expected_text = normalized_text(expected, "fixture label") + if expected_text != candidate_text: + raise RuntimeError( + "output does not exactly match independently known fixture text: " + f"expected={expected_text!r}, actual={candidate_text!r}" + ) + return f"identical normalized text exactly matching {expected_text!r}" + return "identical normalized text; unlabeled fixture, no accuracy claim" + + +def request_json(base_url: str, path: str, payload: dict[str, object]) -> dict: + """POST JSON with a bounded timeout and require an object response.""" + request = urllib.request.Request( + f"{base_url.rstrip('/')}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + return response_json(request) + + +def request_multipart(base_url: str, path: str, model: str, media: bytes) -> dict: + """Upload WAV with deterministic recognition parameters and a collision-checked boundary.""" + if BOUNDARY.encode("ascii") in media: + raise RuntimeError("audio fixture collides with multipart boundary") + body = ( + f"--{BOUNDARY}\r\n" + 'Content-Disposition: form-data; name="model"\r\n\r\n' + f"{model}\r\n" + f"--{BOUNDARY}\r\n" + 'Content-Disposition: form-data; name="response_format"\r\n\r\n' + "json\r\n" + f"--{BOUNDARY}\r\n" + 'Content-Disposition: form-data; name="temperature"\r\n\r\n' + "0\r\n" + f"--{BOUNDARY}\r\n" + 'Content-Disposition: form-data; name="file"; filename="oracle.wav"\r\n' + "Content-Type: audio/wav\r\n\r\n" + ).encode("utf-8") + media + f"\r\n--{BOUNDARY}--\r\n".encode("ascii") + request = urllib.request.Request( + f"{base_url.rstrip('/')}{path}", + data=body, + headers={"Content-Type": f"multipart/form-data; boundary={BOUNDARY}"}, + method="POST", + ) + return response_json(request) + + +def response_json(request: urllib.request.Request) -> dict: + """Read a bounded oracle response and require the JSON object media contract.""" + with urllib.request.urlopen(request, timeout=240) as response: + if response.headers.get_content_type() != "application/json": + raise RuntimeError(f"{request.full_url} returned non-JSON content") + result = json.load(response) + if not isinstance(result, dict): + raise RuntimeError(f"{request.full_url} returned a non-object response") + return result + + +def chat_text(response: dict, source: str) -> object: + """Extract text from exactly one well-formed chat completion choice.""" + choices = response.get("choices") + if not isinstance(choices, list) or len(choices) != 1: + raise RuntimeError(f"{source} returned invalid OCR choices") + choice = choices[0] + if not isinstance(choice, dict): + raise RuntimeError(f"{source} returned an invalid OCR choice") + message = choice.get("message") + if not isinstance(message, dict): + raise RuntimeError(f"{source} returned no OCR message") + return message.get("content") + + +def compare_ocr(candidate_url: str, oracle_url: str, model: str, image: bytes, + expected: str) -> str: + """Compare candidate and oracle OCR with the independent fixture transcription.""" + payload = { + "model": model, + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": OCR_PROMPT}, + {"type": "image_url", "image_url": { + "url": "data:image/png;base64," + base64.b64encode(image).decode("ascii") + }}, + ], + }], + "max_tokens": 64, + "temperature": 0.0, + "seed": 1, + } + candidate = request_json(candidate_url, "/chat/completions", payload) + reference = request_json(oracle_url, "/chat/completions", payload) + return compare_text( + chat_text(candidate, "candidate"), + chat_text(reference, "monolithic reference"), + expected, + ) + + +def compare_asr(candidate_url: str, oracle_url: str, model: str, audio: bytes, + expected: str | None) -> str: + """Compare multipart ASR with the oracle's aligned chat-based audio prompt.""" + # llama-server's /audio/transcriptions substitutes its own default user + # instruction. Compare the actual Skippy audio route with monolithic chat + # using the exact instruction and media ordering that Skippy constructs. + # message_content_to_generation_text joins text/media parts with a newline; + # llama-server concatenates them directly, so preserve that separator here. + candidate = request_multipart(candidate_url, "/audio/transcriptions", model, audio) + reference = request_json(oracle_url, "/chat/completions", { + "model": model, + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": ASR_PROMPT + "\n"}, + {"type": "input_audio", "input_audio": { + "data": base64.b64encode(audio).decode("ascii"), "format": "wav" + }}, + ], + }], + "temperature": 0.0, + "max_tokens": ASR_MAX_TOKENS, + }) + return compare_text( + candidate.get("text"), + chat_text(reference, "monolithic reference"), + expected, + transcript=True, + ) + + +def main() -> None: + """Validate media prerequisites and report class-specific oracle comparison evidence.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidate-url", required=True) + parser.add_argument("--oracle-url", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--class", dest="model_class", required=True, + choices=("ocr", "speech_recognition")) + parser.add_argument("--media-path", required=True, type=Path) + parser.add_argument("--expected-text", help="independent label for the media fixture") + args = parser.parse_args() + + if not args.media_path.is_file(): + parser.error("--media-path must point to a readable fixture") + if args.model_class == "ocr" and args.expected_text is None: + expected = OCR_FIXTURE_TEXT + else: + expected = args.expected_text + media = args.media_path.read_bytes() + if args.model_class == "ocr": + detail = compare_ocr(args.candidate_url, args.oracle_url, args.model, media, expected) + else: + detail = compare_asr(args.candidate_url, args.oracle_url, args.model, media, expected) + print(f"{args.model_class} local-monolithic oracle passed: {detail}") + + +if __name__ == "__main__": + main() diff --git a/scripts/skippy-tts-oracle.py b/scripts/skippy-tts-oracle.py new file mode 100644 index 0000000000..9825b4d574 --- /dev/null +++ b/scripts/skippy-tts-oracle.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python3 +"""Compare deterministic Skippy TTS PCM with pinned llama-tts full-model PCM. + +The public speech endpoint uses a random seed, so this numerical oracle runs a +test-only in-process candidate with shared fixed sampling and the same GGUF, +projector, and prompt as llama-tts. The regular HTTP smoke remains a separate +API contract check; waveform parity does not certify intelligibility. +""" + +from __future__ import annotations + +import argparse +from array import array +import hashlib +import json +import math +import os +from pathlib import Path +import subprocess +import sys +import wave + + +ROOT = Path(__file__).resolve().parents[1] +PROMPT = "The mesh is ready." +SEED = 7 +TOP_K = 20 +TOP_P = 0.8 +# Compare complete utterances; a short frame cap would only compare two +# identically truncated clips and is now rejected by the candidate runtime. +MAX_FRAMES = 512 +CONTEXT_SIZE = 2048 +MAX_RELATIVE_RMS_ERROR = 0.02 +MIN_WAVEFORM_COSINE = 0.9995 +TEST_NAME = "frontend::tests::tts_oracle::deterministic_tts_candidate_when_fixture_is_set" + + +def require_pinned_cpu_oracle(oracle_cli: Path) -> str: + """Verify the CPU reference build options and return its prepared patch identity.""" + if ( + oracle_cli.name != "llama-tts" + or not oracle_cli.is_file() + or not os.access(oracle_cli, os.X_OK) + ): + raise RuntimeError("oracle CLI must be an executable llama-tts binary") + stamp_path = oracle_cli.parent.parent / ".mesh-llm-build-stamp" + patched_sha_path = ROOT / ".deps/llama.cpp/.mesh-llm-patched-sha" + if not stamp_path.is_file() or not patched_sha_path.is_file(): + raise RuntimeError("llama-tts lacks the current pinned build stamp") + stamp = set(stamp_path.read_text(encoding="utf-8").splitlines()) + patched_sha = patched_sha_path.read_text(encoding="utf-8").strip() + required = { + f"patched-sha={patched_sha}", + "backend=cpu", + "link-mode=static", + "ggml-native=OFF", + "cmake-arg=-DGGML_NATIVE=OFF", + "cmake-arg=-DGGML_METAL=OFF", + "cmake-arg=-DLLAMA_BUILD_TOOLS=ON", + } + if not patched_sha or not required.issubset(stamp): + raise RuntimeError("llama-tts lacks the current pinned CPU build stamp") + return patched_sha + + +def require_candidate_cpu_static_build(build_dir: Path, patched_sha: str) -> None: + """Require the candidate's static CPU ABI to match the independent reference.""" + stamp_path = build_dir / ".mesh-llm-build-stamp" + if not stamp_path.is_file(): + raise RuntimeError("TTS candidate lacks a pinned static CPU build stamp") + stamp = set(stamp_path.read_text(encoding="utf-8").splitlines()) + required = { + f"patched-sha={patched_sha}", + "backend=cpu", + "link-mode=static", + "ggml-native=OFF", + "cmake-arg=-DGGML_NATIVE=OFF", + "cmake-arg=-DGGML_METAL=OFF", + } + if not required.issubset(stamp): + raise RuntimeError("TTS candidate static CPU build does not match the oracle patch SHA") + + +def run_logged(command: list[str], log_path: Path, *, env: dict[str, str] | None = None) -> None: + """Bound one oracle process and retain its combined output on success or failure.""" + with log_path.open("w", encoding="utf-8") as log: + try: + result = subprocess.run( + command, + cwd=ROOT, + env=env, + stdout=log, + stderr=subprocess.STDOUT, + timeout=900, + check=False, + ) + except subprocess.TimeoutExpired as error: + raise RuntimeError(f"oracle command timed out; see {log_path}") from error + if result.returncode != 0: + tail = "\n".join(log_path.read_text(encoding="utf-8", errors="replace").splitlines()[-30:]) + raise RuntimeError(f"oracle command exited {result.returncode}; see {log_path}\n{tail}") + + +def read_pcm16_wav(path: Path) -> tuple[int, int, array]: + """Decode nonempty PCM16 WAV data with explicit format and endian validation.""" + try: + with wave.open(str(path), "rb") as audio: + sample_rate = audio.getframerate() + channels = audio.getnchannels() + if audio.getcomptype() != "NONE" or audio.getsampwidth() != 2: + raise RuntimeError(f"{path.name} is not uncompressed PCM16 WAV") + raw = audio.readframes(audio.getnframes()) + except (OSError, EOFError, wave.Error) as error: + raise RuntimeError(f"cannot read {path.name} as WAV: {error}") from error + if sample_rate <= 0 or channels <= 0 or not raw or len(raw) % (2 * channels): + raise RuntimeError(f"{path.name} has invalid or empty PCM samples") + samples = array("h") + samples.frombytes(raw) + if sys.byteorder != "little": + samples.byteswap() + return sample_rate, channels, samples + + +def compare_wavs(candidate_path: Path, oracle_path: Path) -> dict[str, float | int]: + """Check exact audio dimensions and bounded, nonsilent waveform disagreement.""" + candidate_rate, candidate_channels, candidate = read_pcm16_wav(candidate_path) + oracle_rate, oracle_channels, oracle = read_pcm16_wav(oracle_path) + if candidate_rate != oracle_rate or candidate_channels != oracle_channels: + raise RuntimeError("TTS sample rate or channel count differs from monolithic oracle") + if len(candidate) != len(oracle): + raise RuntimeError( + "TTS sample count differs from monolithic oracle: " + f"candidate={len(candidate)}, reference={len(oracle)}" + ) + candidate_energy = math.fsum(float(value) ** 2 for value in candidate) + oracle_energy = math.fsum(float(value) ** 2 for value in oracle) + if candidate_energy <= len(candidate) or oracle_energy <= len(oracle): + raise RuntimeError("TTS candidate or monolithic oracle is silent") + delta_energy = math.fsum( + float(left - right) ** 2 for left, right in zip(candidate, oracle, strict=True) + ) + dot = math.fsum( + float(left) * right for left, right in zip(candidate, oracle, strict=True) + ) + relative_rms_error = math.sqrt(delta_energy / oracle_energy) + waveform_cosine = dot / math.sqrt(candidate_energy * oracle_energy) + metrics: dict[str, float | int] = { + "sample_rate_hz": candidate_rate, + "channels": candidate_channels, + "sample_count": len(candidate) // candidate_channels, + "relative_rms_error": relative_rms_error, + "waveform_cosine": waveform_cosine, + } + if relative_rms_error > MAX_RELATIVE_RMS_ERROR or waveform_cosine < MIN_WAVEFORM_COSINE: + raise RuntimeError( + "TTS PCM differs from monolithic oracle: " + f"relative_rms_error={relative_rms_error:.7g}, waveform_cosine={waveform_cosine:.8g}" + ) + return metrics + + +def sha256(path: Path) -> str: + """Stream an artifact digest for identity-bound certification evidence.""" + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def candidate_test_command(env: dict[str, str]) -> list[str]: + """Select the verified prebuilt test binary or the explicit local test fallback.""" + manifest = env.get("SKIPPY_WORKLOAD_PRODUCER_MANIFEST") + if not manifest: + return ["cargo", "test", "--manifest-path", str(ROOT / "Cargo.toml"), + "-p", "skippy-server", "--lib", TEST_NAME, + "--", "--exact", "--nocapture", "--test-threads=1"] + binary_dir = env.get("SKIPPY_WORKLOAD_CANDIDATE_BIN_DIR") + native_dir = env.get("SKIPPY_WORKLOAD_NATIVE_BUILD_DIR") + if not binary_dir or not native_dir: + raise RuntimeError("prebuilt TTS oracle requires workload candidate and native build paths") + # The deterministic waveform probe must use the same source-bound test + # executable as the class smoke, without rebuilding the canary's Metal tree. + subprocess.run( + [sys.executable, str(ROOT / "scripts/check-skippy-workload-candidate.py"), + "--candidate-binary", str(Path(binary_dir) / "skippy-server"), + "--native-build-dir", native_dir, "--producer-manifest", manifest], + cwd=ROOT, env=env, check=True, + ) + producer = json.loads(Path(manifest).read_text(encoding="utf-8")) + return [producer["files"]["test_binary"]["path"], TEST_NAME, + "--exact", "--nocapture", "--test-threads=1"] + + +def run_oracle(args: argparse.Namespace) -> dict[str, object]: + """Generate both deterministic utterances and bind measured PCM parity to inputs.""" + oracle_cli = Path(args.oracle_cli).resolve() + model_path = Path(args.model_path).resolve() + projector_path = Path(args.projector_path).resolve() + work_dir = Path(args.work_dir).resolve() + patched_sha = require_pinned_cpu_oracle(oracle_cli) + candidate_build_dir = os.environ.get("LLAMA_STAGE_BUILD_DIR") + if not candidate_build_dir: + raise RuntimeError("LLAMA_STAGE_BUILD_DIR is required for pinned TTS candidate execution") + require_candidate_cpu_static_build(Path(candidate_build_dir).resolve(), patched_sha) + for label, path in (("model", model_path), ("projector", projector_path)): + if not path.is_file(): + raise RuntimeError(f"{label} path is not a file: {path}") + if args.layer_end < 1 or not args.model: + raise RuntimeError("model alias and positive layer count are required") + work_dir.mkdir(parents=True, exist_ok=True) + candidate_wav = work_dir / "tts-candidate.wav" + oracle_wav = work_dir / "tts-monolithic-oracle.wav" + result_path = work_dir / "tts-oracle-result.json" + # A failed or filtered test must not inherit a WAV or PASS record from an + # earlier invocation of the same work directory. + for stale_output in (candidate_wav, oracle_wav, result_path): + stale_output.unlink(missing_ok=True) + candidate_env = os.environ.copy() + candidate_env.update({ + "LLAMA_STAGE_BACKEND": "cpu", + "SKIPPY_WORKLOAD_MODEL": str(model_path), + "SKIPPY_WORKLOAD_PROJECTOR": str(projector_path), + "SKIPPY_WORKLOAD_MODEL_ID": args.model, + "SKIPPY_WORKLOAD_LAYER_END": str(args.layer_end), + "SKIPPY_TTS_ORACLE_CANDIDATE_WAV": str(candidate_wav), + "SKIPPY_TTS_ORACLE_PROMPT": PROMPT, + "SKIPPY_TTS_ORACLE_SEED": str(SEED), + "SKIPPY_TTS_ORACLE_TOP_K": str(TOP_K), + "SKIPPY_TTS_ORACLE_TOP_P": str(TOP_P), + "SKIPPY_TTS_ORACLE_MAX_FRAMES": str(MAX_FRAMES), + }) + run_logged( + candidate_test_command(candidate_env), + work_dir / "tts-candidate-test.log", + env=candidate_env, + ) + if not candidate_wav.is_file(): + raise RuntimeError("deterministic TTS candidate test did not write WAV output") + # StageConfig leaves weight repacking disabled. llama-tts enables it by + # default, which changes Q8 logits enough to alter stochastic audio tokens. + run_logged( + [str(oracle_cli), "-m", str(model_path), "-mm", str(projector_path), + "-p", PROMPT, "--output", str(oracle_wav), + "-n", str(MAX_FRAMES), "--seed", str(SEED), + "--top-k", str(TOP_K), "--top-p", str(TOP_P), + "--temp", "1", "--min-p", "0", "--repeat-penalty", "1", + "--no-repack", + "-c", str(CONTEXT_SIZE), "-b", str(CONTEXT_SIZE), + "-ub", str(CONTEXT_SIZE), "-ngl", "0"], + work_dir / "tts-monolithic-oracle.log", + ) + metrics = compare_wavs(candidate_wav, oracle_wav) + result: dict[str, object] = { + "status": "pass", + "class": "speech_synthesis", + "mode": "deterministic_local_monolithic_pcm_parity", + "prompt": PROMPT, + "seed": SEED, + "top_k": TOP_K, + "top_p": TOP_P, + "max_frames": MAX_FRAMES, + "pinned_patch_sha": patched_sha, + "thresholds": { + "max_relative_rms_error": MAX_RELATIVE_RMS_ERROR, + "min_waveform_cosine": MIN_WAVEFORM_COSINE, + }, + "metrics": metrics, + "candidate_wav_sha256": sha256(candidate_wav), + "oracle_wav_sha256": sha256(oracle_wav), + } + result_path.write_text( + json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return result + + +def main() -> None: + """Run the selected TTS comparison and persist its measured evidence.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--oracle-cli", required=True) + parser.add_argument("--model-path", required=True) + parser.add_argument("--projector-path", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--layer-end", type=int, required=True) + parser.add_argument("--work-dir", required=True) + result = run_oracle(parser.parse_args()) + print( + "speech_synthesis local-monolithic oracle passed: " + + json.dumps(result["metrics"], sort_keys=True) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/skippy-workload-certify.sh b/scripts/skippy-workload-certify.sh new file mode 100755 index 0000000000..f7013770b0 --- /dev/null +++ b/scripts/skippy-workload-certify.sh @@ -0,0 +1,390 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +MODEL_CLASS="" +LANE="" +MODEL_PATH="" +MODEL_ID="" +PROJECTOR_PATH="" +WORK_DIR="" +SKIP_BUILD=0 +ORACLE_SERVER="" +ORACLE_COMPLETION="" +ORACLE_TTS="" +ORACLE_REQUIRED=0 +STARTUP_TIMEOUT_SECS=180 + +usage() { + cat >&2 <<'EOF' +usage: scripts/skippy-workload-certify.sh --class CLASS --lane LANE + --model-path PATH --model-id ID --work-dir PATH [--projector-path PATH] + [--oracle-server PATH] [--oracle-completion PATH] [--oracle-tts PATH] + [--require-oracle] # fail closed unless the class-appropriate oracle is selected + [--startup-timeout-secs SECONDS] # per-server readiness deadline (default: 180) + [--skip-build] # oracle runs require a prebuilt SKIPPY_WORKLOAD_PRODUCER_MANIFEST +EOF +} + +while (( $# > 0 )); do + case "$1" in + --class) MODEL_CLASS="$2"; shift ;; + --lane) LANE="$2"; shift ;; + --model-path) MODEL_PATH="$2"; shift ;; + --model-id) MODEL_ID="$2"; shift ;; + --projector-path) PROJECTOR_PATH="$2"; shift ;; + --work-dir) WORK_DIR="$2"; shift ;; + --oracle-server) ORACLE_SERVER="$2"; shift ;; + --oracle-completion) ORACLE_COMPLETION="$2"; shift ;; + --oracle-tts) ORACLE_TTS="$2"; shift ;; + --require-oracle) ORACLE_REQUIRED=1 ;; + --startup-timeout-secs) STARTUP_TIMEOUT_SECS="$2"; shift ;; + --skip-build) SKIP_BUILD=1 ;; + -h|--help) usage; exit 0 ;; + *) echo "unknown option: $1" >&2; usage; exit 1 ;; + esac + shift +done + +if [[ ! "$STARTUP_TIMEOUT_SECS" =~ ^[1-9][0-9]{0,4}$ ]] || (( STARTUP_TIMEOUT_SECS > 86400 )); then + echo "--startup-timeout-secs must be a positive integer between 1 and 86400 seconds" >&2 + exit 1 +fi + +case "$MODEL_CLASS" in + embedding) EXPECTED_LANE="embedding-smoke" ;; + rerank) EXPECTED_LANE="rerank-smoke" ;; + encoder_decoder) EXPECTED_LANE="encoder-decoder-smoke" ;; + ocr) EXPECTED_LANE="ocr-smoke" ;; + speech_synthesis) EXPECTED_LANE="speech-synthesis-smoke" ;; + speech_recognition) EXPECTED_LANE="speech-recognition-smoke" ;; + *) echo "unsupported model class: $MODEL_CLASS" >&2; exit 1 ;; +esac + +if [[ "$LANE" != "$EXPECTED_LANE" ]]; then + echo "lane $LANE does not match class $MODEL_CLASS (expected $EXPECTED_LANE)" >&2 + exit 1 +fi +if [[ -z "$MODEL_PATH" || ! -f "$MODEL_PATH" ]]; then + echo "model path not found: $MODEL_PATH" >&2 + exit 1 +fi +if [[ -z "$MODEL_ID" || -z "$WORK_DIR" ]]; then + echo "--model-id and --work-dir are required" >&2 + exit 1 +fi +if [[ "$MODEL_CLASS" =~ ^(ocr|speech_synthesis|speech_recognition)$ ]] && [[ ! -f "$PROJECTOR_PATH" ]]; then + echo "class $MODEL_CLASS requires a projector path" >&2 + exit 1 +fi +if [[ ( -n "$ORACLE_SERVER" && ( -n "$ORACLE_COMPLETION" || -n "$ORACLE_TTS" ) ) || + ( -n "$ORACLE_COMPLETION" && -n "$ORACLE_TTS" ) ]]; then + echo "select one local-monolithic oracle executable per class" >&2 + exit 1 +fi +if (( ORACLE_REQUIRED == 1 )) && [[ -z "$ORACLE_SERVER" && -z "$ORACLE_COMPLETION" && -z "$ORACLE_TTS" ]]; then + echo "certified workload requires a class-appropriate local-monolithic oracle" >&2 + exit 1 +fi +CANDIDATE_BUILD_DIR="${SKIPPY_WORKLOAD_NATIVE_BUILD_DIR:-${LLAMA_STAGE_BUILD_DIR:-$(LLAMA_STAGE_BACKEND="${LLAMA_STAGE_BACKEND:-cpu}" LLAMA_STAGE_LINK_MODE=static "$ROOT/scripts/build-llama.sh" --print-build-dir)}}" +CANDIDATE_BIN_DIR="${SKIPPY_WORKLOAD_CANDIDATE_BIN_DIR:-$ROOT/target/debug}" +PRODUCER_MANIFEST="${SKIPPY_WORKLOAD_PRODUCER_MANIFEST:-}" +TEST_COMMAND=(cargo test --manifest-path "$ROOT/Cargo.toml" -p skippy-server --lib) +require_pinned_cpu_oracle() { + local executable="$1" expected_name="$2" cmake_option="$3" + local build_dir stamp candidate_build_dir candidate_stamp patched_sha + if [[ ! -x "$executable" ]]; then + echo "oracle executable is not executable: $executable" >&2 + return 1 + fi + patched_sha="$(python3 "$ROOT/scripts/llama-oracle-source.py")" || return 1 + candidate_build_dir="$CANDIDATE_BUILD_DIR" + candidate_stamp="$candidate_build_dir/.mesh-llm-build-stamp" + if [[ ! -f "$candidate_stamp" ]] || + ! grep -Fxq "patched-sha=$patched_sha" "$candidate_stamp" || + ! grep -Fxq 'backend=cpu' "$candidate_stamp" || + ! grep -Fxq 'link-mode=static' "$candidate_stamp" || + ! grep -Fxq 'cmake-arg=-DGGML_METAL=OFF' "$candidate_stamp"; then + echo "candidate static ABI lacks the current pinned CPU llama.cpp build stamp" >&2 + return 1 + fi + build_dir="$(cd "$(dirname "$executable")/.." && pwd -P)" + stamp="$build_dir/.mesh-llm-build-stamp" + if [[ "$(basename "$executable")" != "$expected_name" ]] || + [[ ! -f "$stamp" ]] || + ! grep -Fxq "patched-sha=$patched_sha" "$stamp" || + ! grep -Fxq 'backend=cpu' "$stamp" || + ! grep -Fxq 'cmake-arg=-DGGML_METAL=OFF' "$stamp" || + ! grep -Fxq "$cmake_option" "$stamp"; then + echo "oracle executable lacks the current pinned CPU llama.cpp build stamp" >&2 + return 1 + fi +} +if [[ -n "$ORACLE_SERVER" ]]; then + if [[ ! "$MODEL_CLASS" =~ ^(embedding|rerank|ocr|speech_recognition)$ ]]; then + echo "class $MODEL_CLASS requires a different local-monolithic oracle executable" >&2 + exit 1 + fi + require_pinned_cpu_oracle "$ORACLE_SERVER" llama-server 'cmake-arg=-DLLAMA_BUILD_SERVER=ON' +fi +if [[ -n "$ORACLE_COMPLETION" ]]; then + if [[ "$MODEL_CLASS" != "encoder_decoder" ]]; then + echo "--oracle-completion is only valid for encoder-decoder models" >&2 + exit 1 + fi + require_pinned_cpu_oracle "$ORACLE_COMPLETION" llama-completion 'cmake-arg=-DLLAMA_BUILD_TOOLS=ON' +fi +if [[ -n "$ORACLE_TTS" ]]; then + if [[ "$MODEL_CLASS" != "speech_synthesis" ]]; then + echo "--oracle-tts is only valid for speech synthesis" >&2 + exit 1 + fi + require_pinned_cpu_oracle "$ORACLE_TTS" llama-tts 'cmake-arg=-DLLAMA_BUILD_TOOLS=ON' +fi + +SDK_PYTHON="${SKIPPY_WORKLOAD_SDK_PYTHON:-python3}" +if [[ "$MODEL_CLASS" == "embedding" ]] && ! "$SDK_PYTHON" -c 'import openai' >/dev/null 2>&1; then + echo "official openai-python SDK smoke requires the openai package in $SDK_PYTHON" >&2 + exit 1 +fi + +mkdir -p "$WORK_DIR" +EVIDENCE_PATH="$WORK_DIR/workload-oracle-evidence.json" +COMPARISON_LOG="$WORK_DIR/workload-oracle-comparison.txt" +rm -f "$EVIDENCE_PATH" "$COMPARISON_LOG" +DIMENSIONS="$("$ROOT/scripts/plan-family-battery.py" --inspect-gguf "$MODEL_PATH")" +LAYER_END="$(jq -r '.layer_count' <<<"$DIMENSIONS")" +MODEL_SHA256="$(shasum -a 256 "$MODEL_PATH" | awk '{print $1}')" +N_GPU_LAYERS="${SKIPPY_WORKLOAD_N_GPU_LAYERS:-0}" +BACKEND="${LLAMA_STAGE_BACKEND:-cpu}" +if [[ -n "$PRODUCER_MANIFEST" ]]; then + BACKEND=cpu +fi +if [[ ( -n "$ORACLE_SERVER" || -n "$ORACLE_COMPLETION" || -n "$ORACLE_TTS" ) && + ( "$N_GPU_LAYERS" != "0" || "$BACKEND" != "cpu" ) ]]; then + echo "local-monolithic comparison requires CPU-only candidate execution" >&2 + exit 1 +fi +export LLAMA_STAGE_BACKEND="$BACKEND" +if [[ -n "$ORACLE_SERVER" || -n "$ORACLE_COMPLETION" || -n "$ORACLE_TTS" ]]; then + export LLAMA_STAGE_LINK_MODE=static + export LLAMA_STAGE_BUILD_DIR="$CANDIDATE_BUILD_DIR" +fi + +# The canary explicitly produces a CPU candidate and test binary separately +# from its Metal lane. Consume that immutable, source-bound closure without +# rebuilding or changing the other family lanes' native/Rust outputs. +if [[ -n "$PRODUCER_MANIFEST" ]]; then + python3 "$ROOT/scripts/check-skippy-workload-candidate.py" \ + --candidate-binary "$CANDIDATE_BIN_DIR/skippy-server" \ + --native-build-dir "$CANDIDATE_BUILD_DIR" --producer-manifest "$PRODUCER_MANIFEST" + TEST_COMMAND=("$(jq -er '.files.test_binary.path' "$PRODUCER_MANIFEST")") +elif (( SKIP_BUILD == 0 )); then + LLAMA_STAGE_BUILD_DIR="$CANDIDATE_BUILD_DIR" \ + cargo build -p skippy-server +elif [[ -n "$ORACLE_SERVER" || -n "$ORACLE_COMPLETION" || -n "$ORACLE_TTS" ]]; then + echo "--skip-build oracle certification requires a source-bound workload producer manifest" >&2 + exit 1 +fi +if [[ -n "$ORACLE_SERVER" || -n "$ORACLE_COMPLETION" || -n "$ORACLE_TTS" ]]; then + python3 "$ROOT/scripts/check-skippy-workload-candidate.py" \ + --candidate-binary "$CANDIDATE_BIN_DIR/skippy-server" \ + --native-build-dir "$CANDIDATE_BUILD_DIR" +fi + +MEDIA_PATH="" +case "$MODEL_CLASS" in + ocr) MEDIA_PATH="$ROOT/ci/llama-canary/fixtures/multimodal-smoke.png" ;; + speech_recognition) MEDIA_PATH="$ROOT/ci/llama-canary/fixtures/audio-smoke.wav" ;; +esac + +TEST_FILTER=frontend::tests::non_chat::real_non_chat_class_smoke_when_fixture_is_set +if [[ -n "$PRODUCER_MANIFEST" ]]; then + TEST_COMMAND+=("$TEST_FILTER" --nocapture --exact --test-threads=1) +else + TEST_COMMAND+=("$TEST_FILTER" -- --nocapture --exact --test-threads=1) +fi +env \ + SKIPPY_WORKLOAD_CLASS="$MODEL_CLASS" \ + SKIPPY_WORKLOAD_MODEL="$MODEL_PATH" \ + SKIPPY_WORKLOAD_MODEL_ID="$MODEL_ID" \ + SKIPPY_WORKLOAD_PROJECTOR="$PROJECTOR_PATH" \ + SKIPPY_WORKLOAD_MEDIA="$MEDIA_PATH" \ + SKIPPY_WORKLOAD_LAYER_END="$LAYER_END" \ + SKIPPY_WORKLOAD_CTX_SIZE="${SKIPPY_WORKLOAD_CTX_SIZE:-2048}" \ + SKIPPY_WORKLOAD_MAX_TOKENS="${SKIPPY_WORKLOAD_MAX_TOKENS:-32}" \ + SKIPPY_WORKLOAD_N_GPU_LAYERS="$N_GPU_LAYERS" \ + LLAMA_STAGE_BACKEND="$BACKEND" \ + "${TEST_COMMAND[@]}" + +PORT="${SKIPPY_WORKLOAD_OPENAI_PORT:-19337}" +CONFIG_PATH="$WORK_DIR/stage-openai.json" +python3 - "$CONFIG_PATH" "$MODEL_ID" "$MODEL_PATH" "$MODEL_SHA256" "$LAYER_END" "$N_GPU_LAYERS" "$PROJECTOR_PATH" <<'PY' +import json +import sys + +config_path, model_id, model_path, model_sha256, layer_end, n_gpu_layers, projector_path = sys.argv[1:] +config = { + "run_id": "workload-http-smoke", + "topology_id": "workload-http-smoke-local", + "model_id": model_id, + "model_path": model_path, + "source_model_sha256": model_sha256, + "stage_id": "stage-0", + "stage_index": 0, + "layer_start": 0, + "layer_end": int(layer_end), + "ctx_size": 2048, + "lane_count": 1, + "n_batch": 2048, + "n_ubatch": 2048, + "n_gpu_layers": int(n_gpu_layers), + "selected_device": ({"backend_device": "CPU"} if int(n_gpu_layers) == 0 else None), + "kv_offload": (False if int(n_gpu_layers) == 0 else None), + "op_offload": (False if int(n_gpu_layers) == 0 else None), + "filter_tensors_on_load": False, + "native_mtp_enabled": False, + "load_mode": "runtime-slice", + "bind_addr": "127.0.0.1:0", +} +if projector_path: + config["projector_path"] = projector_path +with open(config_path, "w", encoding="utf-8") as handle: + json.dump(config, handle, indent=2) + handle.write("\n") +PY + +SERVER_LOG="$WORK_DIR/workload-openai-server.log" +LLAMA_STAGE_BACKEND="$BACKEND" \ + "$CANDIDATE_BIN_DIR/skippy-server" serve-openai \ + --config "$CONFIG_PATH" \ + --bind-addr "127.0.0.1:$PORT" \ + --default-max-tokens 128 \ + --telemetry-level off \ + >"$SERVER_LOG" 2>&1 & +SERVER_PID="$!" +ORACLE_PID="" +cleanup() { + if [[ -n "$ORACLE_PID" ]] && kill -0 "$ORACLE_PID" >/dev/null 2>&1; then + kill "$ORACLE_PID" >/dev/null 2>&1 || true + wait "$ORACLE_PID" >/dev/null 2>&1 || true + fi + if kill -0 "$SERVER_PID" >/dev/null 2>&1; then + kill "$SERVER_PID" >/dev/null 2>&1 || true + wait "$SERVER_PID" >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT + +# Both processes receive the same planned wall-clock startup budget, including +# time spent probing the endpoint. An early exit retains the owning log. +wait_for_workload_server() { + local pid="$1" port="$2" log="$3" label="$4" + local deadline=$((SECONDS + STARTUP_TIMEOUT_SECS)) + while (( SECONDS < deadline )); do + if ! kill -0 "$pid" >/dev/null 2>&1; then + echo "$MODEL_CLASS $label exited early" >&2 + tail -80 "$log" >&2 + return 1 + fi + if curl -fsS --max-time 1 "http://127.0.0.1:$port/v1/models" 2>/dev/null \ + | jq -e --arg model "$MODEL_ID" '.data[]? | select(.id == $model)' >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + echo "$MODEL_CLASS $label was not ready within $STARTUP_TIMEOUT_SECS seconds" >&2 + tail -80 "$log" >&2 + return 1 +} +wait_for_workload_server "$SERVER_PID" "$PORT" "$SERVER_LOG" "OpenAI server" +python3 "$ROOT/scripts/ci-openai-workload-smoke.py" \ + --base-url "http://127.0.0.1:$PORT/v1" \ + --model "$MODEL_ID" \ + --class "$MODEL_CLASS" \ + --media-path "$MEDIA_PATH" + +if [[ -n "$ORACLE_SERVER" ]]; then + ORACLE_PORT="${SKIPPY_WORKLOAD_ORACLE_PORT:-19338}" + if [[ ! "$ORACLE_PORT" =~ ^[0-9]+$ ]] || + (( ORACLE_PORT < 1 || ORACLE_PORT > 65535 || ORACLE_PORT == PORT )); then + echo "invalid or conflicting oracle port: $ORACLE_PORT" >&2 + exit 1 + fi + ORACLE_ARGS=( + -m "$MODEL_PATH" -a "$MODEL_ID" --host 127.0.0.1 --port "$ORACLE_PORT" + -c 2048 -b 2048 -ub 2048 -ngl 0 --parallel 1 --no-repack + ) + case "$MODEL_CLASS" in + embedding) ORACLE_ARGS+=(--embedding) ;; + rerank) ORACLE_ARGS+=(--embedding --reranking --pooling rank) ;; + ocr|speech_recognition) ORACLE_ARGS+=(--mmproj "$PROJECTOR_PATH") ;; + esac + ORACLE_LOG="$WORK_DIR/workload-monolithic-oracle-server.log" + "$ORACLE_SERVER" "${ORACLE_ARGS[@]}" >"$ORACLE_LOG" 2>&1 & + ORACLE_PID="$!" + wait_for_workload_server "$ORACLE_PID" "$ORACLE_PORT" "$ORACLE_LOG" "monolithic oracle server" + if [[ "$MODEL_CLASS" =~ ^(ocr|speech_recognition)$ ]]; then + ORACLE_MEDIA_PATH="$MEDIA_PATH" + if [[ "$MODEL_CLASS" == "ocr" ]]; then + ORACLE_MEDIA_PATH="$WORK_DIR/ocr-oracle-mesh-42.png" + python3 "$ROOT/scripts/generate-ocr-oracle-fixture.py" --output "$ORACLE_MEDIA_PATH" + fi + python3 "$ROOT/scripts/skippy-ocr-asr-oracle.py" \ + --candidate-url "http://127.0.0.1:$PORT/v1" \ + --oracle-url "http://127.0.0.1:$ORACLE_PORT/v1" \ + --model "$MODEL_ID" \ + --class "$MODEL_CLASS" \ + --media-path "$ORACLE_MEDIA_PATH" | tee "$COMPARISON_LOG" + else + python3 "$ROOT/scripts/ci-workload-monolithic-oracle.py" \ + --candidate-url "http://127.0.0.1:$PORT/v1" \ + --oracle-url "http://127.0.0.1:$ORACLE_PORT/v1" \ + --model "$MODEL_ID" \ + --class "$MODEL_CLASS" | tee "$COMPARISON_LOG" + fi +fi + +if [[ -n "$ORACLE_COMPLETION" ]]; then + python3 "$ROOT/scripts/ci-workload-monolithic-oracle.py" \ + --candidate-url "http://127.0.0.1:$PORT/v1" \ + --oracle-completion "$ORACLE_COMPLETION" \ + --model-path "$MODEL_PATH" \ + --model "$MODEL_ID" \ + --class "$MODEL_CLASS" | tee "$COMPARISON_LOG" +fi + +if [[ -n "$ORACLE_TTS" ]]; then + python3 "$ROOT/scripts/skippy-tts-oracle.py" \ + --oracle-cli "$ORACLE_TTS" \ + --model-path "$MODEL_PATH" \ + --projector-path "$PROJECTOR_PATH" \ + --model "$MODEL_ID" \ + --layer-end "$LAYER_END" \ + --work-dir "$WORK_DIR" | tee "$COMPARISON_LOG" +fi + +if [[ "$MODEL_CLASS" == "embedding" ]]; then + "$SDK_PYTHON" "$ROOT/scripts/ci-openai-embeddings-smoke.py" \ + --base-url "http://127.0.0.1:$PORT/v1" \ + --model "$MODEL_ID" +fi + +if [[ -n "$ORACLE_SERVER" || -n "$ORACLE_COMPLETION" || -n "$ORACLE_TTS" ]]; then + ORACLE_EXECUTABLE="${ORACLE_SERVER:-${ORACLE_COMPLETION:-$ORACLE_TTS}}" + evidence_command=(python3 "$ROOT/scripts/write-workload-oracle-evidence.py" + --output "$EVIDENCE_PATH" + --comparison-log "$COMPARISON_LOG" + --class "$MODEL_CLASS" + --smoke-lane "$EXPECTED_LANE" + --model-id "$MODEL_ID" + --model-sha256 "$MODEL_SHA256" + --candidate-executable "$CANDIDATE_BIN_DIR/skippy-server" + --oracle-executable "$ORACLE_EXECUTABLE" + --pinned-patch-sha "$(python3 "$ROOT/scripts/llama-oracle-source.py")" + --work-dir "$WORK_DIR") + if [[ -n "$PROJECTOR_PATH" ]]; then + evidence_command+=(--projector-path "$PROJECTOR_PATH") + fi + "${evidence_command[@]}" +fi diff --git a/scripts/skippy-workload-oracles-build.sh b/scripts/skippy-workload-oracles-build.sh new file mode 100644 index 0000000000..b5cfe22819 --- /dev/null +++ b/scripts/skippy-workload-oracles-build.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Explicit producer for the CPU workload candidate and its monolithic oracles. +# Called through just; never touches the canary's Metal/native or Rust outputs. +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PRINT_ENV=0 +if [[ "${1:-}" == "--print-env" ]]; then + PRINT_ENV=1 + shift +fi +if [[ $# != 1 || "$1" != /* || "$1" == *$'\n'* || "$1" == *$'\r'* ]]; then + echo "usage: skippy-workload-oracles-build.sh [--print-env] ABSOLUTE_BUILD_ROOT" >&2 + exit 1 +fi +BUILD_ROOT="$1" +NATIVE_DIR="$BUILD_ROOT/native" +CARGO_DIR="$BUILD_ROOT/cargo" +MANIFEST="$BUILD_ROOT/producer.json" +if (( PRINT_ENV == 1 )); then + printf '%s\n' \ + "SKIPPY_WORKLOAD_ORACLE_SERVER=$NATIVE_DIR/bin/llama-server" \ + "SKIPPY_WORKLOAD_ORACLE_COMPLETION=$NATIVE_DIR/bin/llama-completion" \ + "SKIPPY_WORKLOAD_ORACLE_TTS=$NATIVE_DIR/bin/llama-tts" \ + "SKIPPY_WORKLOAD_NATIVE_BUILD_DIR=$NATIVE_DIR" \ + "SKIPPY_WORKLOAD_CANDIDATE_BIN_DIR=$CARGO_DIR/debug" \ + "SKIPPY_WORKLOAD_PRODUCER_MANIFEST=$MANIFEST" + exit 0 +fi +cd "$ROOT" +mkdir -p "$BUILD_ROOT" +python3 scripts/check-skippy-workload-candidate.py --write-source-snapshot "$BUILD_ROOT/source.json" +python3 scripts/llama-oracle-source.py +export LLAMA_STAGE_BACKEND=cpu LLAMA_STAGE_LINK_MODE=static +export LLAMA_BUILD_DIR="$NATIVE_DIR" LLAMA_STAGE_BUILD_DIR="$NATIVE_DIR" +export LLAMA_STAGE_WORKLOAD_ORACLE=ON LLAMA_STAGE_UPSTREAM_TESTS=OFF +export LLAMA_STAGE_FULL_REPLAY=OFF LLAMA_STAGE_BUILD_TESTS=OFF +export CARGO_TARGET_DIR="$CARGO_DIR" +native_args=() +if [[ "$(uname -s)" == Darwin ]]; then + native_args+=(-DCMAKE_OSX_ARCHITECTURES=arm64) +fi +scripts/build-llama.sh "${native_args[@]}" +just with-lld cargo build --locked -p skippy-server -p skippy-model-package -p skippy-correctness -p skippy-topology --bins +just with-lld cargo test --locked -p skippy-server --lib --no-run --message-format=json > "$BUILD_ROOT/test-artifacts.jsonl" +test_binary="$(jq -rs '[.[] | select(.reason == "compiler-artifact" and .profile.test == true and .target.name == "skippy_server" and .executable != null) | .executable] | unique | if length == 1 then .[0] else error("expected one skippy-server library test binary") end' "$BUILD_ROOT/test-artifacts.jsonl")" +python3 scripts/check-skippy-workload-candidate.py \ + --candidate-binary "$CARGO_DIR/debug/skippy-server" \ + --native-build-dir "$NATIVE_DIR" \ + --test-binary "$test_binary" --write-producer "$MANIFEST" --source-snapshot "$BUILD_ROOT/source.json" diff --git a/scripts/tests/test_check_skippy_workload_candidate.py b/scripts/tests/test_check_skippy_workload_candidate.py new file mode 100644 index 0000000000..e4264c0393 --- /dev/null +++ b/scripts/tests/test_check_skippy_workload_candidate.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import os +import importlib.util +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + + +ROOT = Path(__file__).resolve().parents[2] +CHECK = ROOT / "scripts" / "check-skippy-workload-candidate.py" +SPEC = importlib.util.spec_from_file_location("workload_candidate", CHECK) +assert SPEC and SPEC.loader +CANDIDATE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(CANDIDATE) + + +class CandidateBuildFreshnessTests(unittest.TestCase): + def test_producer_binds_source_native_stamp_and_every_executable(self) -> None: + """Reject producer evidence after source, native build, or executable replacement.""" + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + native = root / "native" + (native / "bin").mkdir(parents=True) + stamp = native / ".mesh-llm-build-stamp" + stamp.write_text("cpu native fixture") + os.utime(stamp, ns=(1, 1)) + binary, test_binary = root / "skippy-server", root / "skippy-tests" + files = CANDIDATE.producer_files(binary, native, test_binary) + for name, path in files.items(): + if name != "native_stamp": + path.write_text(name) + path.chmod(0o755) + source = {"head": "a" * 40, "worktree_sha256": "b" * 64} + snapshot = root / "source.json" + snapshot.write_text(json.dumps(source)) + manifest = root / "producer.json" + with mock.patch.object(CANDIDATE, "source_identity", return_value=source): + CANDIDATE.write_producer(manifest, binary, native, test_binary, snapshot) + CANDIDATE.verify_producer(manifest, binary, native) + for name, path in files.items(): + with self.subTest(artifact=name): + original = path.read_bytes() + path.write_bytes(original + b"tampered") + with self.assertRaisesRegex(RuntimeError, "artifact changed"): + CANDIDATE.verify_producer(manifest, binary, native) + path.write_bytes(original) + source["head"] = "c" * 40 + with self.assertRaisesRegex(RuntimeError, "current repository head"): + CANDIDATE.verify_producer(manifest, binary, native) + with self.assertRaisesRegex(RuntimeError, "source changed while building"): + CANDIDATE.write_producer(manifest, binary, native, test_binary, snapshot) + + def _check(self, binary: Path, build_dir: Path) -> subprocess.CompletedProcess[str]: + """Run the production candidate checker against the isolated build fixture.""" + return subprocess.run( + [ + sys.executable, + str(CHECK), + "--candidate-binary", + str(binary), + "--native-build-dir", + str(build_dir), + ], + capture_output=True, + text=True, + check=False, + ) + + def test_accepts_binary_linked_after_stamped_native_build(self) -> None: + """Accept a candidate linked after its recorded native build.""" + with tempfile.TemporaryDirectory() as temp_dir: + directory = Path(temp_dir) + stamp = directory / ".mesh-llm-build-stamp" + binary = directory / "skippy-server" + stamp.touch() + binary.touch(mode=0o755) + os.utime(stamp, ns=(1_000_000_000, 1_000_000_000)) + os.utime(binary, ns=(2_000_000_000, 2_000_000_000)) + self.assertEqual(0, self._check(binary, directory).returncode) + + def test_rejects_binary_older_than_or_equal_to_native_stamp(self) -> None: + """Reject stale candidates even when their executable paths still exist.""" + with tempfile.TemporaryDirectory() as temp_dir: + directory = Path(temp_dir) + stamp = directory / ".mesh-llm-build-stamp" + binary = directory / "skippy-server" + stamp.touch() + binary.touch(mode=0o755) + os.utime(stamp, ns=(2_000_000_000, 2_000_000_000)) + for binary_time in (1_000_000_000, 2_000_000_000): + with self.subTest(binary_time=binary_time): + os.utime(binary, ns=(binary_time, binary_time)) + result = self._check(binary, directory) + self.assertNotEqual(0, result.returncode) + self.assertIn("candidate executable predates", result.stderr) + + def test_rejects_missing_executable_or_stamp(self) -> None: + """Require the candidate executable and its native-build stamp.""" + with tempfile.TemporaryDirectory() as temp_dir: + directory = Path(temp_dir) + binary = directory / "skippy-server" + self.assertIn("candidate executable is missing", self._check(binary, directory).stderr) + binary.touch(mode=0o755) + self.assertIn("native build stamp is missing", self._check(binary, directory).stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_ci_openai_embeddings_smoke.py b/scripts/tests/test_ci_openai_embeddings_smoke.py new file mode 100644 index 0000000000..1b97886964 --- /dev/null +++ b/scripts/tests/test_ci_openai_embeddings_smoke.py @@ -0,0 +1,167 @@ +"""Reject malformed SDK responses before awarding embedding certification.""" + +from __future__ import annotations + +import base64 +import contextlib +import io +from pathlib import Path +import runpy +import struct +import sys +from types import SimpleNamespace +import unittest +from unittest.mock import Mock, patch + + +SCRIPTS = Path(__file__).resolve().parents[1] +with patch.object(sys, "path", [str(SCRIPTS), *sys.path]): + SMOKE = runpy.run_path(str(SCRIPTS / "ci-openai-embeddings-smoke.py")) + HTTP_SMOKE = runpy.run_path(str(SCRIPTS / "ci-openai-workload-smoke.py")) + + +class EmbeddingSdkSmokeTests(unittest.TestCase): + """Exercise the actual CLI validator with deterministic SDK-shaped responses.""" + + def numeric_response(self) -> SimpleNamespace: + """Supply a valid normalized vector for every canonical smoke input.""" + return SimpleNamespace( + object="list", model="fixture", + data=[SimpleNamespace(object="embedding", index=index, embedding=[1.0, 0.0]) + for index in range(len(SMOKE["EMBEDDING_INPUTS"]))], + usage=SimpleNamespace(prompt_tokens=2), + ) + + def encoded_response(self) -> SimpleNamespace: + """Return the single-input base64 representation of the numeric fixture.""" + return SimpleNamespace( + object="list", model="fixture", + data=[SimpleNamespace(object="embedding", index=0, + embedding=base64.b64encode(struct.pack("<2f", 1, 0)).decode())], + ) + + def run_smoke(self, encoded: SimpleNamespace) -> str: + """Run both SDK requests without a network dependency or installed SDK.""" + client = Mock() + client.embeddings.create.side_effect = [self.numeric_response(), encoded] + sdk = SimpleNamespace(OpenAI=Mock(return_value=client)) + output = io.StringIO() + with patch.dict(sys.modules, {"openai": sdk}), patch.object( + sys, "argv", ["smoke", "--base-url", "http://127.0.0.1:9337/v1", "--model", "fixture"] + ), contextlib.redirect_stdout(output): + SMOKE["main"]() + self.assertEqual(client.embeddings.create.call_count, 2) + self.assertEqual(client.embeddings.create.call_args.kwargs, { + "model": "fixture", "input": SMOKE["EMBEDDING_INPUTS"][0], "encoding_format": "base64", + }) + return output.getvalue() + + def test_valid_single_item_response_passes(self) -> None: + """A correctly labelled, sized and encoded response remains certifiable.""" + self.assertIn("smoke passed", self.run_smoke(self.encoded_response())) + + def test_empty_and_surplus_batches_fail(self) -> None: + """Do not certify a nonempty response that silently adds extra vectors.""" + for size in (0, 2): + with self.subTest(size=size): + response = self.encoded_response() + response.data *= size + with self.assertRaisesRegex(RuntimeError, "wrong batch size"): + self.run_smoke(response) + + def test_response_metadata_must_match_request(self) -> None: + """A correct vector cannot mask the wrong envelope or selected model.""" + for field, value in (("object", "embedding"), ("model", "another-model")): + with self.subTest(field=field): + response = self.encoded_response() + setattr(response, field, value) + with self.assertRaisesRegex(RuntimeError, "wrong object or model"): + self.run_smoke(response) + + def test_item_metadata_must_match_single_input(self) -> None: + """Reject missing, negative and surplus indexes and invalid item objects.""" + for field, value in (("object", "list"), ("object", None), + ("index", -1), ("index", 1), ("index", None), + ("index", False), ("index", 0.0), ("index", "0")): + with self.subTest(field=field, value=value): + response = self.encoded_response() + setattr(response.data[0], field, value) + with self.assertRaisesRegex(RuntimeError, "invalid item metadata"): + self.run_smoke(response) + + def test_invalid_payloads_still_fail(self) -> None: + """Metadata validation must retain type, length and finite-value checks.""" + for payload, message in ( + ([1.0, 0.0], "string"), + (base64.b64encode(struct.pack(" None: + """A finite normalized vector of the right size still needs value parity.""" + response = self.encoded_response() + response.data[0].embedding = base64.b64encode(struct.pack("<2f", 0, 1)).decode() + with self.assertRaisesRegex(RuntimeError, "differs from float"): + self.run_smoke(response) + + def test_base64_parity_allows_float32_rounding(self) -> None: + """Float32 rounding must not reject otherwise equivalent representations.""" + response = self.encoded_response() + response.data[0].embedding = base64.b64encode( + struct.pack("<2f", 1 + 1e-7, 1e-7) + ).decode() + self.run_smoke(response) + + +class EmbeddingHttpSmokeTests(unittest.TestCase): + """The raw HTTP smoke must enforce the same base64 envelope as the SDK smoke.""" + + def run_smoke(self, encoded: dict) -> None: + """Return deterministic related/unrelated vectors before the supplied base64 reply.""" + numeric = {"object": "list", "model": "fixture", "usage": {"prompt_tokens": 3}, + "data": [{"object": "embedding", "index": index, "embedding": vector} + for index, vector in enumerate(([1, 0], [1, 0], [0, 1]))]} + request = Mock(side_effect=[numeric, encoded]) + with patch.dict(HTTP_SMOKE["smoke_embedding"].__globals__, {"request_json": request}): + HTTP_SMOKE["smoke_embedding"]("http://127.0.0.1:9337/v1", "fixture") + + def test_base64_cardinality_and_metadata_are_required(self) -> None: + """A valid vector cannot compensate for a missing, surplus or mislabelled item.""" + item = {"object": "embedding", "index": 0, + "embedding": base64.b64encode(struct.pack("<2f", 1, 0)).decode()} + good = {"object": "list", "model": "fixture", "data": [item]} + self.run_smoke(good) + for changed in ({"data": []}, {"data": [item, item]}, {"data": None}, + {"data": [None]}, {"data": [{**item, "index": 1}]}, + {"data": [{**item, "index": False}]}, + {"data": [{**item, "index": 0.0}]}, + {"data": [{**item, "index": "0"}]}, + {"data": [{**item, "object": None}]}, + {"model": "another-model"}, {"object": "embedding"}): + with self.subTest(changed=changed), self.assertRaises(RuntimeError): + self.run_smoke({**good, **changed}) + + def test_base64_values_must_match_float_response(self) -> None: + """Do not award HTTP certification for a different, correctly sized vector.""" + encoded = {"object": "list", "model": "fixture", "data": [{ + "object": "embedding", "index": 0, + "embedding": base64.b64encode(struct.pack("<2f", 0, 1)).decode(), + }]} + with self.assertRaisesRegex(RuntimeError, "differs from float"): + self.run_smoke(encoded) + + def test_base64_parity_allows_float32_rounding(self) -> None: + """Use the same rounding allowance as the official SDK validator.""" + self.run_smoke({"object": "list", "model": "fixture", "data": [{ + "object": "embedding", "index": 0, + "embedding": base64.b64encode(struct.pack("<2f", 1 + 1e-7, 1e-7)).decode(), + }]}) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_generate_skippy_family_patch.py b/scripts/tests/test_generate_skippy_family_patch.py index 34ac2e1b76..1ecb12e1d1 100644 --- a/scripts/tests/test_generate_skippy_family_patch.py +++ b/scripts/tests/test_generate_skippy_family_patch.py @@ -2,10 +2,9 @@ import importlib.util import json -from pathlib import Path import tempfile import unittest - +from pathlib import Path ROOT = Path(__file__).resolve().parents[2] GENERATOR = ROOT / "scripts/generate-skippy-family-patch.py" @@ -20,6 +19,28 @@ def load_generator(): class GenerateSkippyFamilyPatchTests(unittest.TestCase): + def test_canonical_roster_requires_only_causal_source_mappings(self) -> None: + """Six independently certified non-chat classes must not become decoder shard owners.""" + generator = load_generator() + manifest = ROOT / "ci/llama-canary/family-certified.json" + families = generator.load_certified_families(manifest) + models = json.loads(manifest.read_text())["models"] + self.assertEqual(83, len(families)) + self.assertTrue({"inkling", "llama4"}.issubset(families)) + self.assertEqual({model["family"] for model in models + if model["class"] == "causal_generation"}, families) + + def test_missing_unknown_and_misclassified_workloads_fail_closed(self) -> None: + """An invalid workload class or profile cannot remove a target's mapping requirement.""" + generator = load_generator() + for fields in ({}, {"class": "future"}, {"class": "embedding", "profile": "full"}, + {"class": "causal_generation", "profile": "workload-oracle"}): + with self.subTest(fields=fields), tempfile.TemporaryDirectory() as temporary: + manifest = Path(temporary) / "manifest.json" + manifest.write_text(json.dumps({"models": [{"family": "test", **fields}]})) + with self.assertRaises(RuntimeError): + generator.load_certified_families(manifest) + def test_capture_uses_exact_utf8_bytes_without_newline_translation(self) -> None: generator = load_generator() with tempfile.TemporaryDirectory() as temporary: diff --git a/scripts/tests/test_generate_split_certified.py b/scripts/tests/test_generate_split_certified.py index 5d6f2e1481..448076b117 100644 --- a/scripts/tests/test_generate_split_certified.py +++ b/scripts/tests/test_generate_split_certified.py @@ -4,12 +4,11 @@ import importlib.util import json -from pathlib import Path import subprocess import sys import tempfile import unittest - +from pathlib import Path ROOT = Path(__file__).resolve().parents[2] SCRIPT = ROOT / "scripts" / "generate-split-certified.py" @@ -20,6 +19,27 @@ class SplitCertificationRosterTests(unittest.TestCase): + def test_non_chat_workload_evidence_never_grants_split_admission(self) -> None: + """Only the 83 causal artifacts may enter the host's split admission roster.""" + manifest = json.loads(GENERATOR.DEFAULT_MANIFEST.read_text()) + roster = GENERATOR.build_roster(manifest) + causal = {model["family"] for model in manifest["models"] + if model["class"] == "causal_generation"} + self.assertEqual(83, len(causal)) + self.assertTrue({"inkling", "llama4"}.issubset(causal)) + self.assertEqual(causal, {model["family"] for model in roster["models"]}) + + def test_invalid_workload_class_or_profile_cannot_grant_split_admission(self) -> None: + """Fail closed on absent classes or non-chat rows mislabeled as split-certified.""" + for fields in ({"class": None}, {"class": "future"}, + {"class": "embedding", "profile": "full"}, + {"class": "causal_generation", "profile": "workload-oracle"}): + with self.subTest(fields=fields): + manifest = json.loads(GENERATOR.DEFAULT_MANIFEST.read_text()) + manifest["models"][0].update(fields) + with self.assertRaises(GENERATOR.RosterError): + GENERATOR.build_roster(manifest) + def test_single_file_identity_is_exact_blob_digest(self) -> None: digest = "a" * 64 self.assertEqual( diff --git a/scripts/tests/test_justfile_layout.py b/scripts/tests/test_justfile_layout.py index 450abf35e0..1bf7598a39 100644 --- a/scripts/tests/test_justfile_layout.py +++ b/scripts/tests/test_justfile_layout.py @@ -37,6 +37,7 @@ "bench-corpus", "competitive-benchmark-build", "family-certify", "metrics-server", "metrics-server-build", "skippy-native-full-replay", "skippy-native-tests", "skippy-openai-smoke", + "skippy-workload-oracles-build", "skippy-quantize-build", "skippy-quantize-release-build", "skippy-quantize-standalone-build", "skippy-quantize-standalone-release-build", "skippy-wan-lab-build-bins", diff --git a/scripts/tests/test_llama_native_full_replay.py b/scripts/tests/test_llama_native_full_replay.py index 77b0d6e45d..b7505cccd1 100644 --- a/scripts/tests/test_llama_native_full_replay.py +++ b/scripts/tests/test_llama_native_full_replay.py @@ -30,6 +30,10 @@ "test-skippy-recurrent-state-roundtrip", "test-skippy-verify-checkpoint-retirement", } +NON_CHAT_TARGETS = { + "test-skippy-rerank-template", + "test-skippy-sampling-suppress", +} class LlamaNativeFullReplayTests(unittest.TestCase): @@ -150,17 +154,21 @@ def run_build( return [json.loads(line) for line in trace.read_text().splitlines()] def test_default_build_keeps_standard_and_private_tests_disabled(self) -> None: + """Normal product builds must not silently enable the expensive native certification suite.""" trace = self.run_build(full_replay=False) configure = next(call for call in trace if call["args"][0] != "--build") build = next(call for call in trace if call["args"][0] == "--build") self.assertIn("-DLLAMA_BUILD_TESTS=OFF", configure["args"]) self.assertIn("-DLLAMA_STAGE_BUILD_TESTS=OFF", configure["args"]) + self.assertIn("-DGGML_METAL=OFF", configure["args"]) self.assertTrue(PRIVATE_TARGETS.isdisjoint(build["args"])) self.assertTrue(LEGACY_TARGETS.isdisjoint(build["args"])) + self.assertTrue(NON_CHAT_TARGETS.isdisjoint(build["args"])) self.assertFalse(any(call["tool"] == "ctest" for call in trace)) def test_full_replay_builds_and_runs_only_skippy_gates(self) -> None: + """Explicit full replay must build and execute every retained Skippy gate without upstream tests.""" trace = self.run_build(full_replay=True) configure = next(call for call in trace if call["args"][0] != "--build") build = next(call for call in trace if call["args"][0] == "--build") @@ -172,6 +180,7 @@ def test_full_replay_builds_and_runs_only_skippy_gates(self) -> None: self.assertIn("-DLLAMA_BUILD_SERVER=OFF", configure["args"]) self.assertTrue(PRIVATE_TARGETS.issubset(build["args"])) self.assertTrue(LEGACY_TARGETS.issubset(build["args"])) + self.assertTrue(NON_CHAT_TARGETS.issubset(build["args"])) self.assertNotIn("test-llama-archs", build["args"]) self.assertEqual( [fixture[:4] for fixture in fixtures], diff --git a/scripts/tests/test_llama_oracle_source.py b/scripts/tests/test_llama_oracle_source.py new file mode 100644 index 0000000000..69da36e22f --- /dev/null +++ b/scripts/tests/test_llama_oracle_source.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import hashlib +import importlib.util +from pathlib import Path +import subprocess +import tempfile +import unittest + + +SCRIPT = Path(__file__).resolve().parents[1] / "llama-oracle-source.py" +SPEC = importlib.util.spec_from_file_location("llama_oracle_source", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +source = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(source) + + +class LlamaOracleSourceTests(unittest.TestCase): + def test_digest_includes_generated_series_in_prepare_order(self) -> None: + """Include generated patches in the digest using native preparation order.""" + with tempfile.TemporaryDirectory() as temp_dir: + patches = Path(temp_dir) + (patches / "0001-base.patch").write_bytes(b"base\n") + generated = patches / "generated" + generated.mkdir() + (generated / "series").write_text("0001-family-test.patch\n", encoding="utf-8") + (generated / "0001-family-test.patch").write_bytes(b"generated\n") + expected = hashlib.sha256() + for name, content in ( + ("0001-base.patch", b"base\n"), + ("generated/0001-family-test.patch", b"generated\n"), + ): + expected.update( + f"{name}\n{hashlib.sha256(content).hexdigest()}\n".encode("utf-8") + ) + self.assertEqual(expected.hexdigest(), source.patch_digest(patches)) + (generated / "series").write_text("0002-family-test.patch\n", encoding="utf-8") + with self.assertRaisesRegex(RuntimeError, "invalid generated patch sequence"): + source.patch_digest(patches) + + def test_prepared_checkout_rejects_patch_queue_drift(self) -> None: + """Reject an oracle checkout after its source patch queue changes.""" + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + checkout = root / ".deps/llama.cpp" + checkout.mkdir(parents=True) + patches = root / "third_party/llama.cpp/patches" + patches.mkdir(parents=True) + patch = patches / "0001-test.patch" + patch.write_bytes(b"original\n") + upstream = root / "third_party/llama.cpp/upstream.txt" + upstream.write_text("upstream-sha\n", encoding="utf-8") + subprocess.run(["git", "init", "-q", str(checkout)], check=True) + head = subprocess.run( + ["git", "-C", str(checkout), "rev-parse", "HEAD"], + capture_output=True, text=True, check=False, + ) + if head.returncode != 0: + subprocess.run( + ["git", "-C", str(checkout), "-c", "user.name=Oracle Test", + "-c", "user.email=oracle@example.invalid", "commit", "--allow-empty", + "-qm", "initial"], + check=True, + ) + head_sha = subprocess.run( + ["git", "-C", str(checkout), "rev-parse", "HEAD"], + check=True, capture_output=True, text=True, + ).stdout.strip() + (checkout / ".mesh-llm-upstream-sha").write_text("upstream-sha\n", encoding="utf-8") + (checkout / ".mesh-llm-patch-digest").write_text( + source.patch_digest(patches) + "\n", encoding="utf-8" + ) + (checkout / ".mesh-llm-patched-sha").write_text(head_sha + "\n", encoding="utf-8") + (checkout / ".mesh-llm-prepare-schema").write_text("4\n", encoding="utf-8") + self.assertEqual(head_sha, source.prepared_patched_sha(root)) + patch.write_bytes(b"changed\n") + with self.assertRaisesRegex(RuntimeError, "does not match the current patch queue"): + source.prepared_patched_sha(root) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_llama_upstream_canary_contract.py b/scripts/tests/test_llama_upstream_canary_contract.py index 3d02da12ce..5b43ce2de7 100644 --- a/scripts/tests/test_llama_upstream_canary_contract.py +++ b/scripts/tests/test_llama_upstream_canary_contract.py @@ -3,13 +3,14 @@ import hashlib import json import os -from pathlib import Path import signal import stat +import struct import subprocess import sys import tempfile import unittest +from pathlib import Path ROOT = Path(__file__).resolve().parents[2] WORKFLOW = ROOT / ".github" / "workflows" / "llama-upstream-canary.yml" @@ -543,17 +544,29 @@ def test_timeout_runner_cleans_process_group_when_signalled(self) -> None: stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) - assert wrapper.stdout is not None - child_pid = int(wrapper.stdout.readline()) - wrapper.send_signal(received_signal) - _, stderr = wrapper.communicate(timeout=15) - - self.assertEqual(128 + received_signal, wrapper.returncode) - self.assertIn( - f"signal-fixture received signal {received_signal}", stderr - ) - with self.assertRaises(ProcessLookupError): - os.kill(child_pid, 0) + child_pid = None + try: + assert wrapper.stdout is not None + child_pid = int(wrapper.stdout.readline()) + wrapper.send_signal(received_signal) + _, stderr = wrapper.communicate(timeout=15) + + self.assertEqual(128 + received_signal, wrapper.returncode) + self.assertIn( + f"signal-fixture received signal {received_signal}", stderr + ) + with self.assertRaises(ProcessLookupError): + os.kill(child_pid, 0) + finally: + # A regression must not leave its fixture running on CI. + if wrapper.poll() is None: + wrapper.kill() + if child_pid is not None: + try: + os.killpg(child_pid, signal.SIGKILL) + except ProcessLookupError: + pass + wrapper.communicate(timeout=5) def test_timeout_runner_closes_manifest_stdin_for_children(self) -> None: result = subprocess.run( @@ -580,6 +593,7 @@ def test_timeout_runner_closes_manifest_stdin_for_children(self) -> None: class SkippyFamilyBatteryTests(unittest.TestCase): @staticmethod def _manifest(model: dict[str, object]) -> dict[str, object]: + """Wrap a fixture model in the complete five-profile certification policy.""" return { "schema_version": 1, "policy": { @@ -611,15 +625,27 @@ def _manifest(model: dict[str, object]) -> dict[str, object]: "stage-load", ], }, - } + "workload-smoke": { + "status": "provisional", + "oracle": "none", + "required_lanes": ["class-specific-smoke"], + }, + "workload-oracle": { + "status": "certified", + "oracle": "local-monolithic", + "required_lanes": ["class-specific-smoke", "class-specific-oracle"], + }, + }, }, "models": [model], } @staticmethod def _model(revision: str = "a" * 40) -> dict[str, object]: + """Provide a tiny causal target with immutable artifact identity and no MTP layers.""" return { "family": "test-family", + "class": "causal_generation", "profile": "full", "artifact": { "repo": "org/model", @@ -647,6 +673,7 @@ def _model(revision: str = "a" * 40) -> dict[str, object]: def _dry_run( self, *args: str, models: list[dict[str, object]] | None = None ) -> subprocess.CompletedProcess[str]: + """Exercise the real shell battery against an isolated manifest without native execution.""" with tempfile.TemporaryDirectory() as temp_dir: temp = Path(temp_dir) bin_dir = temp / "bin" @@ -664,6 +691,12 @@ def _dry_run( json.dumps(policy) + "\n", encoding="utf-8" ) env = os.environ.copy() + for key in ( + "SKIPPY_WORKLOAD_ORACLE_SERVER", + "SKIPPY_WORKLOAD_ORACLE_COMPLETION", + "SKIPPY_WORKLOAD_ORACLE_TTS", + ): + env.pop(key, None) env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}" return subprocess.run( [ @@ -698,6 +731,39 @@ def test_battery_builds_once_then_skips_build_in_each_lane(self) -> None: ) ) + def test_workload_dry_run_needs_no_oracle_and_forwards_startup_deadline(self) -> None: + """Keep planning independent of oracle availability while forwarding the startup deadline.""" + model = self._model() + model.update({ + "class": "embedding", + "profile": "workload-oracle", + "evidence": {"fixture": "fixture", "comparison": "fixture"}, + }) + model["execution"]["speculative_policy"] = "disabled" + model["resources"]["startup_timeout_secs"] = 600 + result = self._dry_run("--skip-build", models=[model]) + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn("--startup-timeout-secs 600", result.stdout) + self.assertIn("--require-oracle", result.stdout) + self.assertNotIn("skippy-topology-plan", result.stdout) + self.assertNotIn(str(FAMILY_CERTIFY) + " ", result.stdout) + + def test_mixed_roster_keeps_workload_and_split_certification_separate(self) -> None: + """Execute one distinct lane family per row without staging a non-chat workload.""" + causal = self._model() + workload = self._model() + workload.update({ + "family": "embedding-family", "class": "embedding", "profile": "workload-oracle", + "evidence": {"fixture": "fixture", "comparison": "fixture"}, + }) + workload["execution"]["speculative_policy"] = "disabled" + result = self._dry_run("--skip-build", models=[causal, workload]) + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual(1, result.stdout.count("/skippy-topology-plan ")) + self.assertEqual(1, result.stdout.count(str(FAMILY_CERTIFY) + " ")) + self.assertEqual(1, result.stdout.count("/skippy-workload-certify.sh ")) + self.assertIn("2 certifications planned; no lanes executed", result.stdout) + def test_family_battery_has_no_activation_wire_dtype_switches(self) -> None: script = BATTERY.read_text(encoding="utf-8") @@ -720,6 +786,37 @@ def test_dry_run_reconciles_every_planned_family(self) -> None: self.assertIn("--family test-family", commands[0]) self.assertIn("--family second-family", commands[1]) + def test_supplied_plan_cannot_omit_a_manifest_selected_family(self) -> None: + """Reject a supplied plan that drops a manifest-selected family.""" + with tempfile.TemporaryDirectory() as temp_dir: + temp = Path(temp_dir) + first = self._model() + second = self._model() + second["family"] = "second-family" + manifest = temp / "manifest.json" + policy = self._manifest(first) + policy["models"] = [first, second] + manifest.write_text(json.dumps(policy) + "\n", encoding="utf-8") + generated = subprocess.run( + [str(ROOT / "scripts" / "plan-family-battery.py"), "--manifest", str(manifest)], + cwd=ROOT, text=True, capture_output=True, check=False, + ) + self.assertEqual(0, generated.returncode, generated.stderr) + plan = json.loads(generated.stdout) + plan["selected_models"].pop() + plan["selected_family_count"] = 1 + plan["shards"][0]["families"] = ["test-family"] + supplied = temp / "tampered-plan.json" + supplied.write_text(json.dumps(plan), encoding="utf-8") + result = subprocess.run( + [str(BATTERY), "--manifest", str(manifest), "--plan", str(supplied), + "--dry-run", "--skip-build"], + cwd=ROOT, text=True, capture_output=True, check=False, + ) + self.assertEqual(2, result.returncode) + self.assertIn("differs from the canonical manifest and selection", result.stderr) + self.assertNotIn("model-scans", result.stdout) + def test_family_filter_limits_the_resolved_dry_run(self) -> None: selected = self._dry_run("--families", "test-family") self.assertEqual(0, selected.returncode, selected.stderr) @@ -736,6 +833,7 @@ def test_skip_build_omits_the_one_time_build(self) -> None: self.assertNotIn("cargo build -p skippy-correctness", result.stdout) def test_mmproj_smoke_lane_runs_only_for_families_with_a_projector(self) -> None: + """Only causal rows with a pinned projector schedule the separate multimodal split smoke.""" result = self._dry_run("--skip-build") self.assertEqual(0, result.returncode, result.stderr) self.assertNotIn("mmproj", result.stdout) @@ -761,12 +859,16 @@ def test_mmproj_smoke_lane_runs_only_for_families_with_a_projector(self) -> None self.assertIn("SKIPPY_MM_PROJECTOR=", smokes[0]) self.assertIn("frontend::tests::multimodal", smokes[0]) self.assertIn("--test-threads=1", smokes[0]) - self.assertIn("family battery complete: 1/1", with_mmproj.stdout) + self.assertIn( + "family battery dry run complete: 1 certifications planned; no lanes executed", + with_mmproj.stdout, + ) def test_mmproj_failure_is_accounted_separately_from_core_certification(self) -> None: + """A failed projector smoke must remain visible independently of core parity outcomes.""" script = BATTERY.read_text(encoding="utf-8") smoke_body = script.split("run_mmproj_smoke() {", 1)[1].split( - "\n}\n\nrun_resolved_manifest()", 1 + "\n}\n\nrun_workload_certify()", 1 )[0] self.assertIn("MM_SMOKE_FAILURE_COUNT=0", script) @@ -788,6 +890,7 @@ def test_mmproj_smoke_image_fixture_is_deterministic(self) -> None: ) def test_preflight_pins_snapshot_and_records_native_mtp_models(self) -> None: + """Resolve exact HF snapshots and reject incomplete or mismatched native MTP metadata.""" with tempfile.TemporaryDirectory() as temp_dir: temp = Path(temp_dir) revision = "a" * 40 @@ -801,7 +904,22 @@ def test_preflight_pins_snapshot_and_records_native_mtp_models(self) -> None: / "model.gguf" ) model.parent.mkdir(parents=True) - model.write_bytes(b"gguf-fixture") + def gguf_string(value: str) -> bytes: + """Encode UTF-8 text using the GGUF length-prefixed representation.""" + encoded = value.encode("utf-8") + return struct.pack(" None: check=False, timeout=30, ) - self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) run_dir = next(artifacts.iterdir()) + environment = json.loads( + (run_dir / "preflight" / "environment.json").read_text(encoding="utf-8") + ) + self.assertFalse(environment["port_range"]["checked"]) resolved = (run_dir / "resolved-models.tsv").read_text(encoding="utf-8") self.assertIn(revision, resolved) self.assertIn("|1|1024|5|", resolved) diff --git a/scripts/tests/test_model_artifact_registry.py b/scripts/tests/test_model_artifact_registry.py index 5be398cd01..9d2a2c053a 100644 --- a/scripts/tests/test_model_artifact_registry.py +++ b/scripts/tests/test_model_artifact_registry.py @@ -1,14 +1,13 @@ from __future__ import annotations -import hashlib import fnmatch +import hashlib import json -from pathlib import Path import re import subprocess import tempfile import unittest - +from pathlib import Path ROOT = Path(__file__).resolve().parents[2] REGISTRY = ROOT / "ci" / "model-artifacts" / "registry.json" @@ -18,6 +17,21 @@ class ModelArtifactRegistryTests(unittest.TestCase): + def test_family_schema_covers_every_registered_class_and_profile(self) -> None: + """Published schema enums and required fields must track the canonical workload roster.""" + manifest = json.loads((ROOT / "ci/llama-canary/family-certified.json").read_text()) + schema = json.loads((ROOT / "ci/llama-canary/family-certified.schema.json").read_text()) + model_schema = schema["$defs"]["model"] + self.assertIn("class", model_schema["required"]) + self.assertEqual({row["class"] for row in manifest["models"]}, + set(model_schema["properties"]["class"]["enum"])) + self.assertEqual(set(manifest["policy"]["profiles"]), + set(model_schema["properties"]["profile"]["enum"])) + profiles = schema["properties"]["policy"]["properties"]["profiles"] + self.assertEqual(set(manifest["policy"]["profiles"]), set(profiles["required"])) + self.assertEqual({"fixture", "comparison"}, + set(model_schema["properties"]["evidence"]["required"])) + def test_generated_manifests_are_current(self) -> None: subprocess.run(["python3", str(GENERATOR), "--check"], cwd=ROOT, check=True) diff --git a/scripts/tests/test_plan_family_battery.py b/scripts/tests/test_plan_family_battery.py index 6d0dc2496a..7ec683cf2e 100644 --- a/scripts/tests/test_plan_family_battery.py +++ b/scripts/tests/test_plan_family_battery.py @@ -2,12 +2,11 @@ import copy import json -from pathlib import Path import struct import subprocess import tempfile import unittest - +from pathlib import Path ROOT = Path(__file__).resolve().parents[2] PLANNER = ROOT / "scripts" / "plan-family-battery.py" @@ -110,14 +109,31 @@ def _run( ) def test_checked_in_policy_resolves_all_certified_models(self) -> None: + """Preserve all primary split targets and six explicitly classed workload-oracle rows.""" result = self._run() self.assertEqual(0, result.returncode, result.stderr) plan = json.loads(result.stdout) - self.assertEqual(83, plan["selected_family_count"]) + self.assertEqual(89, plan["selected_family_count"]) self.assertEqual( ["single-step", "chain", "state-handoff"], plan["required_certification_lanes"], ) + self.assertEqual( + { + "causal_generation", + "embedding", + "rerank", + "encoder_decoder", + "ocr", + "speech_synthesis", + "speech_recognition", + }, + {model["class"] for model in plan["selected_models"]}, + ) + self.assertEqual( + ["embedding-smoke", "embedding-oracle"], + plan["model_class_lanes"]["embedding"], + ) glm47 = next( model for model in plan["selected_models"] @@ -141,6 +157,27 @@ def test_checked_in_policy_resolves_all_certified_models(self) -> None: qwen4exp = by_family["qwen4exp"] self.assertEqual(10240, qwen4exp["execution"]["activation_width"]) self.assertEqual(3, len(qwen4exp["artifact"]["files"])) + expected_workloads = { + "nomic-bert-embedding": ("embedding", "embedding-smoke"), + "jina-bert-v2-rerank": ("rerank", "rerank-smoke"), + "t5-encoder-decoder": ( + "encoder_decoder", + "encoder-decoder-smoke", + ), + "paddleocr": ("ocr", "ocr-smoke"), + "qwen3tts": ("speech_synthesis", "speech-synthesis-smoke"), + "ultravox": ("speech_recognition", "speech-recognition-smoke"), + } + for family, (model_class, lane) in expected_workloads.items(): + with self.subTest(family=family): + model = by_family[family] + self.assertEqual(model_class, model["class"]) + self.assertEqual([lane, lane.replace("-smoke", "-oracle")], model["certification_lanes"]) + self.assertEqual("workload-oracle", model["profile"]) + self.assertEqual("certified", model["certification_status"]) + self.assertEqual("local-monolithic", model["oracle"]) + self.assertEqual("disabled", model["execution"]["speculative_policy"]) + self.assertEqual(0, model["execution"]["mtp_layers"]) for auxiliary in ("deepseek4", "gemma4-assistant", "muse-glimmer", "glm-dsa"): self.assertNotIn(auxiliary, by_family) @@ -150,6 +187,7 @@ def test_cadence_selection_is_removed(self) -> None: self.assertIn("unrecognized arguments: --cadence", result.stderr) def test_mmproj_artifacts_resolve_and_cover_the_vision_families(self) -> None: + """Require immutable projector sidecars for the complete causal and non-chat media roster.""" result = self._run() self.assertEqual(0, result.returncode, result.stderr) plan = json.loads(result.stdout) @@ -166,6 +204,9 @@ def test_mmproj_artifacts_resolve_and_cover_the_vision_families(self) -> None: "qwen2-vl", "qwen3-vl", "qwen3vlmoe", + "paddleocr", + "qwen3tts", + "ultravox", }, set(with_mmproj), ) @@ -176,9 +217,7 @@ def test_mmproj_artifacts_resolve_and_cover_the_vision_families(self) -> None: self.assertEqual( set(mmproj["file_integrity"]), set(mmproj["files"]) ) - self.assertRegex( - mmproj["files"][0], r"^mmproj" - ) + self.assertIn("mmproj", mmproj["files"][0].lower()) def test_certified_model_requires_an_explicit_activation_width(self) -> None: manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) @@ -214,6 +253,82 @@ def test_certified_profile_cannot_add_or_reorder_core_lanes(self) -> None: self.assertEqual(2, result.returncode) self.assertIn("must require exactly the three core lanes", result.stderr) + def test_workload_smoke_profile_cannot_claim_an_oracle(self) -> None: + """Prevent smoke profiles from claiming independent-oracle evidence.""" + manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) + manifest["policy"]["profiles"]["workload-smoke"]["oracle"] = "local-monolithic" + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "manifest.json" + path.write_text(json.dumps(manifest), encoding="utf-8") + result = self._run(path) + self.assertEqual(2, result.returncode) + self.assertIn("workload-smoke must remain provisional and oracle-free", result.stderr) + + def test_workload_oracle_profile_cannot_drop_the_oracle_lane(self) -> None: + """Require the oracle lane for profiles claiming oracle validation.""" + manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) + manifest["policy"]["profiles"]["workload-oracle"]["required_lanes"] = ["class-specific-smoke"] + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "manifest.json" + path.write_text(json.dumps(manifest), encoding="utf-8") + result = self._run(path) + self.assertEqual(2, result.returncode) + self.assertIn("workload-oracle requires certified local-monolithic", result.stderr) + + def test_certified_workload_requires_fixture_and_comparison_evidence(self) -> None: + """Require fixtures and comparison evidence before workload certification.""" + manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) + model = next(row for row in manifest["models"] if row["profile"] == "workload-oracle") + del model["evidence"] + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "manifest.json" + path.write_text(json.dumps(manifest), encoding="utf-8") + result = self._run(path) + self.assertEqual(2, result.returncode) + self.assertIn("evidence must be an object", result.stderr) + + def test_supplied_plan_rejects_tampered_selected_model_rows(self) -> None: + """Reject changes to canonical model selection or shard metadata in supplied plans.""" + generated = self._run(MANIFEST, "--shard-count", "2") + self.assertEqual(0, generated.returncode, generated.stderr) + plan = json.loads(generated.stdout) + family = plan["selected_models"].pop()["family"] + plan["selected_family_count"] -= 1 + for shard in plan["shards"]: + if family in shard["families"]: + shard["families"].remove(family) + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "tampered-plan.json" + path.write_text(json.dumps(plan), encoding="utf-8") + result = self._run(MANIFEST, "--verify-plan", str(path)) + self.assertEqual(2, result.returncode) + self.assertIn("differs from the canonical manifest and selection", result.stderr) + + def test_supplied_plan_rejects_inflated_oracle_status(self) -> None: + """Reject plans promoting uncertified manifest entries to oracle-certified status.""" + generated = self._run(MANIFEST, "--families", "nomic-bert-embedding") + self.assertEqual(0, generated.returncode, generated.stderr) + plan = json.loads(generated.stdout) + plan["selected_models"][0]["oracle"] = "none" + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "tampered-plan.json" + path.write_text(json.dumps(plan), encoding="utf-8") + result = self._run(MANIFEST, "--verify-plan", str(path)) + self.assertEqual(2, result.returncode) + self.assertIn("differs from the canonical manifest and selection", result.stderr) + + def test_non_chat_model_cannot_claim_the_certified_full_profile(self) -> None: + """Prevent non-chat workloads from inheriting causal-only certification.""" + manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) + model = next(row for row in manifest["models"] if row["class"] == "embedding") + model["profile"] = "full" + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "manifest.json" + path.write_text(json.dumps(manifest), encoding="utf-8") + result = self._run(path) + self.assertEqual(2, result.returncode) + self.assertIn("requires a class-specific workload profile", result.stderr) + def test_duplicate_family_is_rejected(self) -> None: manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) manifest["models"].append(copy.deepcopy(manifest["models"][0])) @@ -224,6 +339,98 @@ def test_duplicate_family_is_rejected(self) -> None: self.assertEqual(2, result.returncode) self.assertIn("duplicate family", result.stderr) + def test_model_class_is_required_and_selects_class_specific_lanes(self) -> None: + """Require an explicit supported model class and its corresponding validation lanes.""" + source = json.loads(MANIFEST.read_text(encoding="utf-8")) + model = copy.deepcopy(source["models"][0]) + source["models"] = [model] + del model["class"] + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "manifest.json" + path.write_text(json.dumps(source), encoding="utf-8") + missing = self._run(path) + model["class"] = "embedding" + model["profile"] = "workload-smoke" + model["execution"]["speculative_policy"] = "disabled" + path.write_text(json.dumps(source), encoding="utf-8") + embedding = self._run(path) + + self.assertEqual(2, missing.returncode) + self.assertIn("models[0].class must be", missing.stderr) + self.assertEqual(0, embedding.returncode, embedding.stderr) + selected = json.loads(embedding.stdout)["selected_models"][0] + self.assertEqual("embedding", selected["class"]) + self.assertEqual(["embedding-smoke"], selected["certification_lanes"]) + + def test_projector_classes_require_an_explicit_projector_artifact(self) -> None: + """Require a pinned projector for model classes that consume media.""" + source = json.loads(MANIFEST.read_text(encoding="utf-8")) + model = copy.deepcopy(source["models"][0]) + source["models"] = [model] + model["class"] = "speech_recognition" + model["profile"] = "workload-smoke" + model["execution"]["speculative_policy"] = "disabled" + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "manifest.json" + path.write_text(json.dumps(source), encoding="utf-8") + result = self._run(path) + + self.assertEqual(2, result.returncode) + self.assertIn("requires an mmproj_artifact", result.stderr) + + def test_non_causal_classes_reject_split_and_speculative_policy(self) -> None: + """Reject split and speculative policies unsupported by non-causal workloads.""" + source = json.loads(MANIFEST.read_text(encoding="utf-8")) + model = copy.deepcopy(source["models"][0]) + source["models"] = [model] + model["class"] = "embedding" + model["profile"] = "workload-smoke" + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "manifest.json" + path.write_text(json.dumps(source), encoding="utf-8") + speculative = self._run(path) + model["execution"]["speculative_policy"] = "disabled" + model["execution"]["mtp_layers"] = 1 + path.write_text(json.dumps(source), encoding="utf-8") + split = self._run(path) + + self.assertEqual(2, speculative.returncode) + self.assertIn("must disable speculative decoding", speculative.stderr) + self.assertEqual(2, split.returncode) + self.assertIn("must not request split or MTP certification", split.stderr) + + def test_inspect_gguf_reports_canonical_dimensions_without_a_manifest(self) -> None: + """Read canonical GGUF dimensions independently of family-manifest metadata.""" + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "fixture.gguf" + self._write_gguf(path, 7, 1536) + result = subprocess.run( + [str(PLANNER), "--inspect-gguf", str(path)], + cwd=ROOT, + text=True, + capture_output=True, + check=False, + ) + + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual( + {"activation_width": 1536, "layer_count": 7}, + json.loads(result.stdout), + ) + + def test_unknown_model_class_is_rejected(self) -> None: + """Reject unknown model classes instead of selecting a causal fallback.""" + source = json.loads(MANIFEST.read_text(encoding="utf-8")) + source["models"] = [copy.deepcopy(source["models"][0])] + source["models"][0]["class"] = "guessed-from-name" + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "manifest.json" + path.write_text(json.dumps(source), encoding="utf-8") + result = self._run(path) + + self.assertEqual(2, result.returncode) + self.assertIn("class must be one of", result.stderr) + def test_cache_gate_requires_every_exact_revision_file(self) -> None: source = json.loads(MANIFEST.read_text(encoding="utf-8")) source["models"] = [copy.deepcopy(source["models"][0])] @@ -495,6 +702,7 @@ def test_cache_gate_requires_dimensions_in_at_least_one_shard(self) -> None: self.assertIn("has no GGUF shard", result.stderr) def test_shards_are_deterministic_and_preserve_every_family_once(self) -> None: + """Sharding must be reproducible and neither duplicate nor omit a selected family.""" first = self._run(MANIFEST, "--shard-count", "4") second = self._run(MANIFEST, "--shard-count", "4") self.assertEqual(0, first.returncode, first.stderr) @@ -503,8 +711,8 @@ def test_shards_are_deterministic_and_preserve_every_family_once(self) -> None: families = [ family for shard in plan["shards"] for family in shard["families"] ] - self.assertEqual(83, len(families)) - self.assertEqual(83, len(set(families))) + self.assertEqual(89, len(families)) + self.assertEqual(89, len(set(families))) self.assertEqual(4, len(plan["github_matrix"]["include"])) diff --git a/scripts/tests/test_release_notes.py b/scripts/tests/test_release_notes.py index fdbc93c1c7..6127c3525a 100644 --- a/scripts/tests/test_release_notes.py +++ b/scripts/tests/test_release_notes.py @@ -1105,10 +1105,12 @@ def test_link_only_body_is_recovered_classified_and_published(self): tracked.write_text("base\n", encoding="utf-8") subprocess.run(["git", "add", "tracked"], cwd=directory, check=True) subprocess.run(["git", "commit", "-qm", "chore: base"], cwd=directory, check=True) - subprocess.run(["git", "tag", "v1"], cwd=directory, check=True) + # These local fixture tags are lightweight even when the developer's + # global Git config defaults to signing tags and opening an editor. + subprocess.run(["git", "tag", "--no-sign", "v1"], cwd=directory, check=True) tracked.write_text("fixed\n", encoding="utf-8") subprocess.run(["git", "commit", "-qam", "fix: recover the release entry"], cwd=directory, check=True) - subprocess.run(["git", "tag", "v2"], cwd=directory, check=True) + subprocess.run(["git", "tag", "--no-sign", "v2"], cwd=directory, check=True) fake_bin = directory / "bin" fake_bin.mkdir() diff --git a/scripts/tests/test_run_command_with_timeout.py b/scripts/tests/test_run_command_with_timeout.py new file mode 100644 index 0000000000..e78dd65566 --- /dev/null +++ b/scripts/tests/test_run_command_with_timeout.py @@ -0,0 +1,96 @@ +"""Deterministic signal-boundary regressions for the canary process supervisor.""" + +from __future__ import annotations + +import argparse +import importlib.util +from pathlib import Path +import signal +import subprocess +import unittest +from unittest import mock + + +SOURCE = Path(__file__).resolve().parents[1] / "run-command-with-timeout.py" +SPEC = importlib.util.spec_from_file_location("timeout_runner", SOURCE) +assert SPEC is not None and SPEC.loader is not None +RUNNER = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(RUNNER) + + +class TimeoutSignalSafetyTests(unittest.TestCase): + def exercise_signal_boundary(self, boundary: str, signum: int) -> None: + """Inject cancellation at a boundary without sending signals to the test runner.""" + handlers = {signal.SIGINT: signal.SIG_DFL, signal.SIGTERM: signal.SIG_DFL} + events: list[str] = [] + in_spawn = False + in_wait = False + process = mock.Mock() + + def install_handler(number, handler): + previous = handlers[number] + handlers[number] = handler + return previous + + def request_cancel(number): + self.assertTrue(callable(handlers[number]), "handler must be installed before spawn") + handlers[number](number, None) + + def spawn(*_args, **_kwargs): + nonlocal in_spawn + in_spawn = True + try: + if boundary == "spawn": + request_cancel(signum) + finally: + in_spawn = False + events.append("spawned") + return process + + def wait(*_args, **kwargs): + nonlocal in_wait + in_wait = True + try: + request_cancel(signum) + raise subprocess.TimeoutExpired("fixture", kwargs["timeout"]) + finally: + in_wait = False + + def cleanup(actual_process): + self.assertIs(process, actual_process) + self.assertFalse(in_spawn, "cleanup needs the completed Popen object") + self.assertFalse(in_wait, "cleanup must not reenter Popen.wait from a signal handler") + # Repeated cancellation during cleanup must neither reenter cleanup + # nor replace the signal that determined the wrapper's exit status. + request_cancel(signal.SIGINT if signum == signal.SIGTERM else signal.SIGTERM) + events.append("cleaned") + + process.wait.side_effect = wait + args = argparse.Namespace(seconds=30, label="fixture", command=["fixture"]) + with ( + mock.patch.object(RUNNER, "parse_args", return_value=args), + mock.patch.object(RUNNER.signal, "signal", side_effect=install_handler), + mock.patch.object(RUNNER.subprocess, "Popen", side_effect=spawn) as popen, + mock.patch.object(RUNNER, "terminate_group", side_effect=cleanup) as terminate, + ): + self.assertEqual(128 + signum, RUNNER.main()) + popen.assert_called_once_with( + ["fixture"], stdin=subprocess.DEVNULL, start_new_session=True, + ) + terminate.assert_called_once_with(process) + self.assertEqual(["spawned", "cleaned"], events) + self.assertEqual({signal.SIGINT: signal.SIG_DFL, signal.SIGTERM: signal.SIG_DFL}, handlers) + + def test_cancellation_during_spawn_waits_for_child_ownership(self) -> None: + for signum in (signal.SIGINT, signal.SIGTERM): + with self.subTest(signum=signum): + self.exercise_signal_boundary("spawn", signum) + + def test_cancellation_during_wait_defers_cleanup_until_wait_unwinds(self) -> None: + for signum in (signal.SIGINT, signal.SIGTERM): + with self.subTest(signum=signum): + self.exercise_signal_boundary("wait", signum) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_runtime_events_native_gate.py b/scripts/tests/test_runtime_events_native_gate.py index f08ef81c3e..610e5b8127 100644 --- a/scripts/tests/test_runtime_events_native_gate.py +++ b/scripts/tests/test_runtime_events_native_gate.py @@ -339,9 +339,31 @@ def test_gate_model_resolves_at_every_workflow_cadence(self) -> None: self.assertEqual(len(resolved["sha256"]), 64) self.assertGreater(int(resolved["size_bytes"]), 0) + def test_missing_gate_model_cadence_still_fails_closed(self) -> None: + """The gate must reject a fixture not authorized for its event cadence.""" + inputs = self.steps["Restore runtime-event gate model"]["with"] + source = json.loads((ROOT / inputs["model_manifest"]).read_text()) + for cadence in ("pull-request", "main", "manual"): + with self.subTest(cadence=cadence), tempfile.TemporaryDirectory() as directory: + manifest = json.loads(json.dumps(source)) + artifact = next(row for row in manifest["artifacts"] + if row["id"] == inputs["model_artifact_id"]) + artifact["cadences"].remove(cadence) + path = Path(directory) / "manifest.json" + path.write_text(json.dumps(manifest)) + result = subprocess.run( + ["python3", str(ROOT / "scripts/resolve-test-model-manifest.py"), + str(path), "--artifact-id", inputs["model_artifact_id"], + "--cadence", cadence, "--require-single-file"], + cwd=ROOT, text=True, capture_output=True, check=False, + ) + self.assertEqual(result.returncode, 2) + self.assertIn("is not allowed at cadence", result.stderr) + def test_family_certification_has_one_complete_model_list(self) -> None: + """The full family roster is independent of ordinary CI artifact cadence authorization.""" manifest = json.loads((ROOT / "ci/llama-canary/family-certified.json").read_text()) - self.assertEqual(83, len(manifest["models"])) + self.assertEqual(89, len(manifest["models"])) self.assertTrue(all("cadences" not in model for model in manifest["models"])) def test_evidence_is_uploaded_even_when_the_gate_fails(self) -> None: diff --git a/scripts/tests/test_skippy_ocr_asr_oracle.py b/scripts/tests/test_skippy_ocr_asr_oracle.py new file mode 100644 index 0000000000..d869cddd1b --- /dev/null +++ b/scripts/tests/test_skippy_ocr_asr_oracle.py @@ -0,0 +1,152 @@ +"""Regression tests for deterministic OCR media and multimodal parity checks.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +import struct +import unittest +from unittest.mock import patch +import zlib + + +ROOT = Path(__file__).resolve().parents[2] + + +def import_script(name: str, path: Path): + """Load an oracle helper directly without starting its command-line entrypoint.""" + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +fixture = import_script("ocr_oracle_fixture", ROOT / "scripts" / "generate-ocr-oracle-fixture.py") +oracle = import_script("skippy_ocr_asr_oracle", ROOT / "scripts" / "skippy-ocr-asr-oracle.py") + + +class OcrFixtureTests(unittest.TestCase): + def test_png_is_deterministic_and_text_bearing(self): + """The OCR label must be represented by stable image pixels, not random media.""" + png = fixture.png_bytes() + self.assertEqual(png, fixture.png_bytes()) + self.assertTrue(png.startswith(b"\x89PNG\r\n\x1a\n")) + self.assertEqual(fixture.TEXT, "MESH 42") + size = struct.unpack(">II", png[16:24]) + self.assertEqual(size, (fixture.WIDTH, fixture.HEIGHT)) + + image_data_start = png.index(b"IDAT") + 4 + compressed_length = struct.unpack(">I", png[image_data_start - 8:image_data_start - 4])[0] + raw = zlib.decompress(png[image_data_start:image_data_start + compressed_length]) + self.assertEqual(len(raw), fixture.HEIGHT * (1 + fixture.WIDTH * 3)) + self.assertIn(b"\x00\x00\x00", raw) + self.assertIn(b"\xff\xff\xff", raw) + + +class MultimodalOracleTests(unittest.TestCase): + def test_ocr_requires_both_parity_and_known_text(self): + """Matching outputs are insufficient unless they also equal the independent label.""" + self.assertIn("mesh 42", oracle.compare_text("MESH 42", "Mesh 42.", "MESH 42")) + with self.assertRaisesRegex(RuntimeError, "differs from monolithic"): + oracle.compare_text("MESH 42", "MESH 43", "MESH 42") + with self.assertRaisesRegex(RuntimeError, "does not exactly match independently known"): + oracle.compare_text("unrelated", "Unrelated.", "MESH 42") + with self.assertRaisesRegex(RuntimeError, "does not exactly match independently known"): + oracle.compare_text("MESH 42 extra", "Mesh 42 extra.", "MESH 42") + + def test_matching_incorrect_text_cannot_pass_by_containing_the_label(self): + """Extra hallucinated text must not pass an OCR substring match.""" + for text in ("not mesh 42", "mesh 42 unrelated text", "mesh 42 mesh 42"): + with self.subTest(text=text), self.assertRaisesRegex(RuntimeError, "does not exactly match independently known"): + oracle.compare_text(text, text, "MESH 42") + + def test_asr_unlabeled_fixture_does_not_claim_accuracy(self): + """Reference agreement without ground truth remains parity, not recognition accuracy.""" + detail = oracle.compare_text("The mesh is ready.", "the mesh is ready", None) + self.assertIn("no accuracy claim", detail) + with self.assertRaisesRegex(RuntimeError, "returned empty text"): + oracle.compare_text("!", "?", None) + + def test_asr_normalizes_only_known_decorative_prefixes(self): + """Normalization must not erase arbitrary transcript prefixes to manufacture agreement.""" + detail = oracle.compare_text( + 'The text is: "The mesh is ready"', + 'The audio is: "The mesh is ready"', + None, + transcript=True, + ) + self.assertIn("identical normalized text", detail) + with self.assertRaisesRegex(RuntimeError, "differs from monolithic"): + oracle.compare_text("The text is: mesh ready", "The audio is: mesh not ready", + None, transcript=True) + with self.assertRaisesRegex(RuntimeError, "differs from monolithic"): + oracle.compare_text("Transcribed text: mesh ready", "The audio is: mesh ready", + None, transcript=True) + + def test_asr_matching_refusals_do_not_pass_as_transcripts(self): + """Identical refusals are failures even when reference and candidate agree.""" + with self.assertRaisesRegex(RuntimeError, "not a transcript"): + oracle.compare_text("I can't fulfill this request.", + "I can't fulfill this request.", None, transcript=True) + with self.assertRaisesRegex(RuntimeError, "not a transcript"): + oracle.compare_text("I can help you with transcribing audio to text.", + "I can help you with transcribing audio to text.", + None, transcript=True) + + def test_asr_repeated_reference_content_is_not_normalized_away(self): + """Repeated recognized content must remain visible to the equivalence comparison.""" + with self.assertRaisesRegex(RuntimeError, "differs from monolithic"): + oracle.compare_text("The mesh is ready", "The mesh is ready. The mesh is ready.", + None, transcript=True) + + def test_ocr_sends_same_request_to_both_servers(self): + """Candidate and reference receive identical model, image and prompt inputs.""" + reply = {"choices": [{"message": {"content": "MESH 42"}}]} + with patch.object(oracle, "request_json", side_effect=[reply, reply]) as request: + oracle.compare_ocr("http://candidate/v1", "http://reference/v1", "ocr", b"png", "MESH 42") + self.assertEqual(request.call_count, 2) + candidate_call, reference_call = request.call_args_list + self.assertEqual(candidate_call.args[1:], reference_call.args[1:]) + self.assertEqual(candidate_call.args[1], "/chat/completions") + + def test_asr_aligns_reference_chat_prompt_with_candidate_audio_route(self): + """Compare equivalent transcription tasks across the two different HTTP interfaces.""" + with ( + patch.object(oracle, "request_multipart", return_value={"text": "The mesh is ready."}) as candidate, + patch.object(oracle, "request_json", return_value={ + "choices": [{"message": {"content": "The mesh is ready."}}] + }) as reference, + ): + oracle.compare_asr("http://candidate/v1", "http://reference/v1", "asr", b"wav", None) + self.assertEqual(candidate.call_args.args, + ("http://candidate/v1", "/audio/transcriptions", "asr", b"wav")) + self.assertEqual(reference.call_args.args[:2], + ("http://reference/v1", "/chat/completions")) + payload = reference.call_args.args[2] + parts = payload["messages"][0]["content"] + self.assertEqual(parts[0]["text"], oracle.ASR_PROMPT + "\n") + self.assertEqual(parts[1]["input_audio"]["data"], "d2F2") + self.assertEqual(payload["max_tokens"], oracle.ASR_MAX_TOKENS) + # The multipart API has no max_tokens field; set the candidate server's + # default explicitly instead of comparing its CLI default of 16 to 128. + runner = (ROOT / "scripts/skippy-workload-certify.sh").read_text() + self.assertIn(f"--default-max-tokens {oracle.ASR_MAX_TOKENS}", runner) + + def test_asr_multipart_contains_deterministic_fields_and_audio(self): + """The upload preserves exact media bytes and deterministic transcription options.""" + with patch.object(oracle, "response_json", return_value={"text": "ok"}) as response: + oracle.request_multipart("http://localhost/v1/", "/audio/transcriptions", "asr", b"RIFF\x00audio") + request = response.call_args.args[0] + self.assertEqual(request.full_url, "http://localhost/v1/audio/transcriptions") + body = request.data + self.assertIn(b'name="model"\r\n\r\nasr\r\n', body) + self.assertIn(b'name="response_format"\r\n\r\njson\r\n', body) + self.assertIn(b'name="temperature"\r\n\r\n0\r\n', body) + self.assertIn(b"RIFF\x00audio", body) + with self.assertRaisesRegex(RuntimeError, "collides with multipart boundary"): + oracle.request_multipart("http://localhost/v1", "/audio/transcriptions", "asr", + oracle.BOUNDARY.encode("ascii")) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_skippy_static_link.py b/scripts/tests/test_skippy_static_link.py new file mode 100644 index 0000000000..8938d4bd25 --- /dev/null +++ b/scripts/tests/test_skippy_static_link.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +BUILD_SCRIPT = ROOT / "crates/skippy-ffi/build.rs" +CORE_ARCHIVES = ( + "src/libllama.a", + "common/libllama-common.a", + "common/libllama-common-base.a", + "ggml/src/libggml.a", + "ggml/src/libggml-base.a", + "ggml/src/libggml-cpu.a", +) +OPTIONAL_ARCHIVES = ( + "ggml/src/ggml-blas/libggml-blas.a", + "ggml/src/ggml-cuda/libggml-cuda.a", + "ggml/src/ggml-hip/libggml-hip.a", + "ggml/src/ggml-vulkan/libggml-vulkan.a", + "ggml/src/ggml-metal/libggml-metal.a", +) + + +class SkippyStaticLinkTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + """Compile the real build script once for isolated backend-selection tests.""" + cls.binary_dir = tempfile.TemporaryDirectory() + cls.addClassCleanup(cls.binary_dir.cleanup) + cls.binary = Path(cls.binary_dir.name) / "build-script" + result = subprocess.run( + ["rustc", "--edition=2024", str(BUILD_SCRIPT), "-o", str(cls.binary)], + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + if result.returncode: + raise RuntimeError(f"build script fixture failed: {result.stderr}") + + def _run(self, backend: str, flags: dict[str, str]) -> subprocess.CompletedProcess[str]: + """Execute a backend-selection fixture with Unix-style CMake cache lines.""" + return self._run_with_newline(backend, flags, "\n") + + def _run_with_newline( + self, backend: str, flags: dict[str, str], newline: str + ) -> subprocess.CompletedProcess[str]: + """Run the build script with isolated archives and controlled CMake cache line endings.""" + fixture = tempfile.TemporaryDirectory() + self.addCleanup(fixture.cleanup) + build_dir = Path(fixture.name) / "native" + for relative in (*CORE_ARCHIVES, *OPTIONAL_ARCHIVES): + archive = build_dir / relative + archive.parent.mkdir(parents=True, exist_ok=True) + archive.touch() + (build_dir / "CMakeCache.txt").write_text( + "".join(f"{key}:BOOL={value}{newline}" for key, value in flags.items()), + encoding="utf-8", + newline="", + ) + env = { + key: value for key, value in os.environ.items() + if not key.startswith(("LLAMA_STAGE_", "SKIPPY_LLAMA_", "CARGO_FEATURE_")) + } + env.update({ + "CARGO_MANIFEST_DIR": str(ROOT / "crates/skippy-ffi"), + "TARGET": "aarch64-apple-darwin", + "LLAMA_STAGE_BACKEND": backend, + "LLAMA_STAGE_BUILD_DIR": str(build_dir), + "LLAMA_STAGE_LINK_MODE": "static", + "SKIPPY_LLAMA_AUTO_BUILD": "0", + }) + return subprocess.run( + [str(self.binary)], cwd=ROOT, env=env, + capture_output=True, text=True, check=False, + ) + + def test_cpu_ignores_stale_gpu_and_blas_archives(self) -> None: + """Keep CPU linking independent of stale accelerator archives.""" + result = self._run("cpu", { + "GGML_BLAS": "OFF", "GGML_CUDA": "OFF", "GGML_HIP": "OFF", + "GGML_VULKAN": "OFF", "GGML_METAL": "OFF", + }) + self.assertEqual(0, result.returncode, result.stderr) + for library in ("ggml-blas", "ggml-cuda", "ggml-hip", "ggml-vulkan", "ggml-metal"): + self.assertNotIn(f"cargo:rustc-link-lib=static={library}", result.stdout) + for framework in ("Foundation", "Metal", "MetalKit"): + self.assertNotIn(f"cargo:rustc-link-lib=framework={framework}", result.stdout) + + def test_active_metal_backend_links_only_cache_enabled_archive(self) -> None: + """Link only the selected Metal archive enabled by CMake.""" + result = self._run("metal", {"GGML_METAL": "ON", "GGML_BLAS": "ON"}) + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn("cargo:rustc-link-lib=static=ggml-metal", result.stdout) + self.assertIn("cargo:rustc-link-lib=framework=Metal", result.stdout) + self.assertIn("cargo:rustc-link-lib=static=ggml-blas", result.stdout) + self.assertNotIn("cargo:rustc-link-lib=static=ggml-cuda", result.stdout) + + def test_backend_cache_mismatch_fails_closed_despite_stale_archive(self) -> None: + """Reject a disabled selected backend even when a stale archive exists.""" + result = self._run("metal", {"GGML_METAL": "OFF"}) + self.assertNotEqual(0, result.returncode) + self.assertIn("selected backend requires GGML_METAL=ON", result.stderr) + + def test_unselected_backend_cache_mismatch_fails_closed(self) -> None: + """Reject an accelerator enabled outside the selected native backend.""" + result = self._run("cpu", {"GGML_CUDA": "ON"}) + self.assertNotEqual(0, result.returncode) + self.assertIn("staged backend mismatch: GGML_CUDA=ON", result.stderr) + + def test_crlf_cache_values_are_recognized(self) -> None: + """Recognize CMake booleans written with Windows line endings.""" + result = self._run_with_newline("metal", {"GGML_METAL": "ON"}, "\r\n") + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn("cargo:rustc-link-lib=static=ggml-metal", result.stdout) + + def test_enabled_staged_accelerator_requires_matching_selected_backend(self) -> None: + """Require the runtime backend to match enabled staged accelerators.""" + for key in ("GGML_CUDA", "GGML_HIP", "GGML_VULKAN", "GGML_METAL"): + with self.subTest(key=key): + result = self._run("cpu", {key: "ON"}) + self.assertNotEqual(0, result.returncode) + self.assertIn(f"staged backend mismatch: {key}=ON", result.stderr) + + def test_cache_boolean_accepts_crlf_and_surrounding_whitespace(self) -> None: + """Normalize line endings and whitespace when reading CMake booleans.""" + for value in ("ON\r", " TRUE \r", " 1 "): + with self.subTest(value=value): + result = self._run("metal", {"GGML_METAL": value}) + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn("cargo:rustc-link-lib=static=ggml-metal", result.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_skippy_tts_oracle.py b/scripts/tests/test_skippy_tts_oracle.py new file mode 100644 index 0000000000..3c9dd87e51 --- /dev/null +++ b/scripts/tests/test_skippy_tts_oracle.py @@ -0,0 +1,231 @@ +from __future__ import annotations + +import argparse +from array import array +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest import mock +import wave + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts" / "skippy-tts-oracle.py" +SPEC = importlib.util.spec_from_file_location("skippy_tts_oracle", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +oracle = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(oracle) + + +def write_wav(path: Path, samples: list[int], *, rate: int = 8000) -> None: + """Create controlled PCM fixtures with explicit waveform and format properties.""" + pcm = array("h", samples) + if sys.byteorder != "little": + pcm.byteswap() + with wave.open(str(path), "wb") as output: + output.setnchannels(1) + output.setsampwidth(2) + output.setframerate(rate) + output.writeframes(pcm.tobytes()) + + +class TtsOracleTests(unittest.TestCase): + def test_prebuilt_candidate_requires_verified_producer_and_never_runs_cargo(self) -> None: + """A prebuilt lane must verify its producer instead of silently rebuilding a candidate.""" + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + manifest = root / "producer.json" + test_binary = str(root / "candidate tests") + manifest.write_text(json.dumps({"files": {"test_binary": {"path": test_binary}}})) + env = { + "SKIPPY_WORKLOAD_PRODUCER_MANIFEST": str(manifest), + "SKIPPY_WORKLOAD_CANDIDATE_BIN_DIR": str(root / "bin"), + "SKIPPY_WORKLOAD_NATIVE_BUILD_DIR": str(root / "native"), + } + with mock.patch.object(oracle.subprocess, "run") as verify: + command = oracle.candidate_test_command(env) + self.assertEqual([test_binary, oracle.TEST_NAME, "--exact", "--nocapture", "--test-threads=1"], command) + verify.assert_called_once_with( + [sys.executable, str(ROOT / "scripts/check-skippy-workload-candidate.py"), + "--candidate-binary", str(root / "bin/skippy-server"), + "--native-build-dir", str(root / "native"), "--producer-manifest", str(manifest)], + cwd=ROOT, env=env, check=True, + ) + with mock.patch.object(oracle.subprocess, "run", side_effect=subprocess.CalledProcessError(1, ["verify"])): + with self.assertRaises(subprocess.CalledProcessError): + oracle.candidate_test_command(env) + + def test_incomplete_producer_paths_do_not_fall_back_to_cargo(self) -> None: + """Partial producer configuration is rejected rather than replaced by ambient builds.""" + with self.assertRaisesRegex(RuntimeError, "requires workload candidate and native build paths"): + oracle.candidate_test_command({"SKIPPY_WORKLOAD_PRODUCER_MANIFEST": "producer.json"}) + + def test_standalone_candidate_retains_explicit_cargo_test(self) -> None: + """Standalone mode retains its explicit native candidate test invocation.""" + command = oracle.candidate_test_command({}) + self.assertEqual(["cargo", "test"], command[:2]) + self.assertIn(oracle.TEST_NAME, command) + + def test_oracle_invocation_matches_candidate_no_repack_context(self) -> None: + """Align reference context size and repacking policy with the candidate.""" + with tempfile.TemporaryDirectory() as temp_dir: + test_root = Path(temp_dir) + model = test_root / "model.gguf" + projector = test_root / "projector.gguf" + model.write_bytes(b"fixture") + projector.write_bytes(b"fixture") + work_dir = test_root / "evidence" + commands: list[list[str]] = [] + + def fake_run_logged(command: list[str], _log_path: Path, **_kwargs: object) -> None: + """Emulate independent TTS execution while preserving its logged-output contract.""" + commands.append(command) + name = "tts-candidate.wav" if command[0] == "cargo" else "tts-monolithic-oracle.wav" + write_wav(work_dir / name, [1000, -1000] * 800) + + args = argparse.Namespace( + oracle_cli=str(test_root / "bin" / "llama-tts"), + model_path=str(model), + projector_path=str(projector), + model="qwen3-tts", + layer_end=28, + work_dir=str(work_dir), + ) + with mock.patch.dict(os.environ, {"LLAMA_STAGE_BUILD_DIR": str(test_root / "abi")}, clear=True): + with mock.patch.object(oracle, "require_pinned_cpu_oracle", return_value="pinned-sha"): + with mock.patch.object(oracle, "require_candidate_cpu_static_build"): + with mock.patch.object(oracle, "run_logged", side_effect=fake_run_logged): + result = oracle.run_oracle(args) + + self.assertEqual("pass", result["status"]) + self.assertEqual(2, len(commands)) + self.assertIn("--no-repack", commands[1]) + self.assertEqual("0", commands[1][commands[1].index("--min-p") + 1]) + self.assertEqual("0", commands[1][commands[1].index("-ngl") + 1]) + + def test_identical_pcm_passes_with_zero_error(self) -> None: + """Identical waveforms establish the comparator's exact-agreement baseline.""" + with tempfile.TemporaryDirectory() as temp_dir: + candidate = Path(temp_dir) / "candidate.wav" + reference = Path(temp_dir) / "reference.wav" + samples = [1000 if index % 2 else -1000 for index in range(1600)] + write_wav(candidate, samples) + write_wav(reference, samples) + metrics = oracle.compare_wavs(candidate, reference) + self.assertEqual(0.0, metrics["relative_rms_error"]) + self.assertEqual(1.0, metrics["waveform_cosine"]) + self.assertEqual(1600, metrics["sample_count"]) + + def test_small_pcm_rounding_difference_is_tolerated(self) -> None: + """Allow bounded quantization noise without accepting arbitrary waveform changes.""" + with tempfile.TemporaryDirectory() as temp_dir: + candidate = Path(temp_dir) / "candidate.wav" + reference = Path(temp_dir) / "reference.wav" + write_wav(candidate, [1001, -999] * 800) + write_wav(reference, [1000, -1000] * 800) + metrics = oracle.compare_wavs(candidate, reference) + self.assertLess(metrics["relative_rms_error"], oracle.MAX_RELATIVE_RMS_ERROR) + + def test_wrong_gain_and_phase_are_rejected(self) -> None: + """Amplitude or phase changes must fail even when duration and format match.""" + with tempfile.TemporaryDirectory() as temp_dir: + candidate = Path(temp_dir) / "candidate.wav" + reference = Path(temp_dir) / "reference.wav" + write_wav(reference, [1000, -1000] * 800) + for changed in ([1060, -1060] * 800, [-1000, 1000] * 800): + with self.subTest(changed=changed[:2]): + write_wav(candidate, changed) + with self.assertRaisesRegex(RuntimeError, "differs from monolithic oracle"): + oracle.compare_wavs(candidate, reference) + + def test_silent_and_mismatched_wav_are_rejected(self) -> None: + """Reject silence and incompatible waveform metadata before comparing samples.""" + with tempfile.TemporaryDirectory() as temp_dir: + candidate = Path(temp_dir) / "candidate.wav" + reference = Path(temp_dir) / "reference.wav" + write_wav(reference, [1000, -1000] * 800) + write_wav(candidate, [0] * 1600) + with self.assertRaisesRegex(RuntimeError, "silent"): + oracle.compare_wavs(candidate, reference) + write_wav(candidate, [1000, -1000] * 799) + with self.assertRaisesRegex(RuntimeError, "sample count differs"): + oracle.compare_wavs(candidate, reference) + write_wav(candidate, [1000, -1000] * 800, rate=16000) + with self.assertRaisesRegex(RuntimeError, "sample rate"): + oracle.compare_wavs(candidate, reference) + + def test_oracle_requires_current_pinned_cpu_stamp(self) -> None: + """Stale or mismatched reference builds cannot certify the current source.""" + with tempfile.TemporaryDirectory() as temp_dir: + test_root = Path(temp_dir) + binary = test_root / "build" / "bin" / "llama-tts" + binary.parent.mkdir(parents=True) + binary.write_bytes(b"binary") + binary.chmod(0o755) + patched_sha = test_root / ".deps" / "llama.cpp" / ".mesh-llm-patched-sha" + patched_sha.parent.mkdir(parents=True) + patched_sha.write_text("pinned-sha\n", encoding="utf-8") + stamp = test_root / "build" / ".mesh-llm-build-stamp" + with mock.patch.object(oracle, "ROOT", test_root): + with self.assertRaisesRegex(RuntimeError, "pinned build stamp"): + oracle.require_pinned_cpu_oracle(binary) + stamp.write_text( + "patched-sha=pinned-sha\nbackend=cpu\nlink-mode=static\n" + "ggml-native=OFF\ncmake-arg=-DGGML_NATIVE=OFF\n" + "cmake-arg=-DGGML_METAL=OFF\n" + "cmake-arg=-DLLAMA_BUILD_TOOLS=ON\n", + encoding="utf-8", + ) + oracle.require_pinned_cpu_oracle(binary) + stamp.write_text( + "patched-sha=pinned-sha\nbackend=cpu\nlink-mode=static\n" + "ggml-native=ON\ncmake-arg=-DGGML_NATIVE=ON\n" + "cmake-arg=-DGGML_METAL=OFF\n" + "cmake-arg=-DLLAMA_BUILD_TOOLS=ON\n", + encoding="utf-8", + ) + with self.assertRaisesRegex(RuntimeError, "pinned CPU build stamp"): + oracle.require_pinned_cpu_oracle(binary) + stamp.write_text("patched-sha=pinned-sha\nbackend=metal\n", encoding="utf-8") + with self.assertRaisesRegex(RuntimeError, "pinned CPU build stamp"): + oracle.require_pinned_cpu_oracle(binary) + + def test_candidate_requires_same_pinned_cpu_static_build(self) -> None: + """Candidate and reference must agree on the pinned CPU native build identity.""" + with tempfile.TemporaryDirectory() as temp_dir: + build_dir = Path(temp_dir) + stamp = build_dir / ".mesh-llm-build-stamp" + with self.assertRaisesRegex(RuntimeError, "lacks a pinned static CPU build stamp"): + oracle.require_candidate_cpu_static_build(build_dir, "pinned-sha") + stamp.write_text( + "patched-sha=old-sha\nbackend=cpu\nlink-mode=static\n" + "ggml-native=OFF\ncmake-arg=-DGGML_NATIVE=OFF\n" + "cmake-arg=-DGGML_METAL=OFF\n", + encoding="utf-8", + ) + with self.assertRaisesRegex(RuntimeError, "does not match"): + oracle.require_candidate_cpu_static_build(build_dir, "pinned-sha") + stamp.write_text( + "patched-sha=pinned-sha\nbackend=cpu\nlink-mode=static\n" + "ggml-native=OFF\ncmake-arg=-DGGML_NATIVE=OFF\n" + "cmake-arg=-DGGML_METAL=OFF\n", + encoding="utf-8", + ) + oracle.require_candidate_cpu_static_build(build_dir, "pinned-sha") + stamp.write_text( + "patched-sha=pinned-sha\nbackend=cpu\nlink-mode=static\n" + "ggml-native=ON\ncmake-arg=-DGGML_NATIVE=ON\n" + "cmake-arg=-DGGML_METAL=OFF\n", + encoding="utf-8", + ) + with self.assertRaisesRegex(RuntimeError, "does not match"): + oracle.require_candidate_cpu_static_build(build_dir, "pinned-sha") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_skippy_workload_certify.py b/scripts/tests/test_skippy_workload_certify.py new file mode 100644 index 0000000000..f4fe5eb7b8 --- /dev/null +++ b/scripts/tests/test_skippy_workload_certify.py @@ -0,0 +1,191 @@ +from __future__ import annotations + +from pathlib import Path +import os +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +RUNNER = ROOT / "scripts" / "skippy-workload-certify.sh" + + +class WorkloadCertifyContractTests(unittest.TestCase): + def _run(self, *args: str) -> subprocess.CompletedProcess[str]: + """Invoke the certification wrapper and retain its status and diagnostics for assertions.""" + return subprocess.run( + [str(RUNNER), *args], + cwd=ROOT, + text=True, + capture_output=True, + check=False, + ) + + def test_help_documents_the_typed_certification_inputs(self) -> None: + """Keep the public wrapper usage aligned with its class-specific inputs.""" + result = self._run("--help") + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn("--class CLASS", result.stderr) + self.assertIn("--lane LANE", result.stderr) + self.assertIn("--projector-path PATH", result.stderr) + self.assertIn("--oracle-server PATH", result.stderr) + self.assertIn("--oracle-completion PATH", result.stderr) + self.assertIn("--oracle-tts PATH", result.stderr) + self.assertIn("--startup-timeout-secs SECONDS", result.stderr) + self.assertIn("--require-oracle", result.stderr) + self.assertIn("--startup-timeout-secs", result.stderr) + + def test_startup_deadline_rejects_invalid_values_before_execution(self) -> None: + """Malformed or out-of-range budgets must not reach native startup.""" + for value in ("0", "-1", "1.5", "01", "86401", "abc"): + with self.subTest(value=value): + result = self._run("--startup-timeout-secs", value) + self.assertEqual(1, result.returncode) + self.assertIn("must be a positive integer", result.stderr) + + def test_missing_embedding_sdk_fails_before_model_execution(self) -> None: + """Embedding certification cannot proceed without its required official SDK lane.""" + with tempfile.TemporaryDirectory() as temp_dir: + model = Path(temp_dir) / "model.gguf" + model.touch() + result = subprocess.run( + [str(RUNNER), "--class", "embedding", "--lane", "embedding-smoke", + "--model-path", str(model), "--model-id", "fixture", "--work-dir", temp_dir, + "--skip-build"], cwd=ROOT, text=True, capture_output=True, check=False, + env={**os.environ, "SKIPPY_WORKLOAD_SDK_PYTHON": str(Path(temp_dir) / "missing-python")}, + ) + self.assertEqual(1, result.returncode) + self.assertIn("official openai-python SDK smoke requires", result.stderr) + self.assertFalse((Path(temp_dir) / "workload-oracle-evidence.json").exists()) + + def test_startup_timeout_must_be_positive(self) -> None: + """A zero startup budget is a usage error rather than an immediate runtime failure.""" + result = self._run("--startup-timeout-secs", "0") + self.assertEqual(1, result.returncode) + self.assertIn("must be a positive integer", result.stderr) + + def test_embedding_sdk_smoke_cannot_be_skipped(self) -> None: + """Keep SDK verification mandatory instead of accepting a skipped lane as success.""" + runner = RUNNER.read_text(encoding="utf-8") + self.assertIn("official openai-python SDK smoke requires", runner) + self.assertNotIn("SDK smoke skipped", runner) + + def test_certified_mode_rejects_missing_oracle_before_build(self) -> None: + """Certified mode requires the independent reference before consuming build resources.""" + with tempfile.TemporaryDirectory() as temp_dir: + model = Path(temp_dir) / "model.gguf" + model.touch() + result = self._run( + "--class", "embedding", "--lane", "embedding-smoke", + "--model-path", str(model), "--model-id", "fixture", + "--work-dir", temp_dir, "--require-oracle", "--skip-build", + ) + self.assertEqual(1, result.returncode) + self.assertIn("certified workload requires a class-appropriate", result.stderr) + + def test_unknown_class_fails_before_model_execution(self) -> None: + """The wrapper must reject classes outside its closed workload vocabulary.""" + result = self._run("--class", "guessed", "--lane", "guessed-equivalence") + self.assertEqual(1, result.returncode) + self.assertIn("unsupported model class: guessed", result.stderr) + + def test_lane_must_match_the_selected_class(self) -> None: + """A mismatched lane label cannot produce evidence for a different workload.""" + result = self._run( + "--class", + "embedding", + "--lane", + "rerank-smoke", + ) + self.assertEqual(1, result.returncode) + self.assertIn("expected embedding-smoke", result.stderr) + + def test_projector_classes_fail_closed_without_a_projector(self) -> None: + """OCR and speech certification require explicit projector inputs.""" + with tempfile.TemporaryDirectory() as temp_dir: + model = Path(temp_dir) / "model.gguf" + model.touch() + for model_class, lane in ( + ("ocr", "ocr-smoke"), + ("speech_synthesis", "speech-synthesis-smoke"), + ("speech_recognition", "speech-recognition-smoke"), + ): + with self.subTest(model_class=model_class): + result = self._run( + "--class", + model_class, + "--lane", + lane, + "--model-path", + str(model), + "--model-id", + "fixture", + "--work-dir", + temp_dir, + "--skip-build", + ) + self.assertEqual(1, result.returncode) + self.assertIn("requires a projector path", result.stderr) + + def test_oracle_requires_a_pinned_cpu_server_binary(self) -> None: + """A model file or other nonexecutable cannot masquerade as the reference server.""" + with tempfile.TemporaryDirectory() as temp_dir: + model = Path(temp_dir) / "model.gguf" + model.touch() + result = self._run( + "--class", "embedding", "--lane", "embedding-smoke", + "--model-path", str(model), "--model-id", "fixture", + "--work-dir", temp_dir, "--oracle-server", str(model), + "--skip-build", + ) + self.assertEqual(1, result.returncode) + self.assertIn("oracle executable is not executable", result.stderr) + + def test_speech_synthesis_rejects_server_oracle(self) -> None: + """Speech synthesis requires the dedicated waveform reference, not an HTTP server.""" + with tempfile.TemporaryDirectory() as temp_dir: + model = Path(temp_dir) / "model.gguf" + projector = Path(temp_dir) / "projector.gguf" + model.touch() + projector.touch() + result = self._run( + "--class", "speech_synthesis", "--lane", "speech-synthesis-smoke", + "--model-path", str(model), "--projector-path", str(projector), + "--model-id", "fixture", "--work-dir", temp_dir, + "--oracle-server", str(model), "--skip-build", + ) + self.assertEqual(1, result.returncode) + self.assertIn("requires a different local-monolithic oracle", result.stderr) + + def test_other_classes_reject_tts_oracle(self) -> None: + """The waveform executable cannot certify embedding or other server workloads.""" + with tempfile.TemporaryDirectory() as temp_dir: + model = Path(temp_dir) / "model.gguf" + model.touch() + result = self._run( + "--class", "embedding", "--lane", "embedding-smoke", + "--model-path", str(model), "--model-id", "fixture", + "--work-dir", temp_dir, "--oracle-tts", str(model), + "--skip-build", + ) + self.assertEqual(1, result.returncode) + self.assertIn("only valid for speech synthesis", result.stderr) + + def test_encoder_decoder_requires_direct_completion_oracle(self) -> None: + """Encoder-decoder certification must use the direct completion reference.""" + with tempfile.TemporaryDirectory() as temp_dir: + model = Path(temp_dir) / "model.gguf" + model.touch() + result = self._run( + "--class", "encoder_decoder", "--lane", "encoder-decoder-smoke", + "--model-path", str(model), "--model-id", "fixture", + "--work-dir", temp_dir, "--oracle-server", str(model), + "--skip-build", + ) + self.assertEqual(1, result.returncode) + self.assertIn("requires a different local-monolithic oracle", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_skippy_workload_oracles_build.py b/scripts/tests/test_skippy_workload_oracles_build.py new file mode 100644 index 0000000000..4b87d39abf --- /dev/null +++ b/scripts/tests/test_skippy_workload_oracles_build.py @@ -0,0 +1,60 @@ +"""CPU producer contract shared by unchanged and changed-pin canaries.""" +import json +import subprocess +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +PRODUCER = ROOT / "scripts/skippy-workload-oracles-build.sh" + + +class WorkloadOracleProducerTests(unittest.TestCase): + def test_environment_is_deterministic_and_does_not_override_metal_outputs(self) -> None: + """The CPU oracle closure must not overwrite the primary Metal family outputs.""" + result = subprocess.run( + ["bash", str(PRODUCER), "--print-env", "/tmp/canary with spaces"], + text=True, capture_output=True, check=True, + ) + values = dict(line.split("=", 1) for line in result.stdout.splitlines()) + self.assertEqual(6, len(values)) + self.assertTrue(all(name.startswith("SKIPPY_WORKLOAD_") for name in values)) + for suffix, executable in [("SERVER", "llama-server"), ("COMPLETION", "llama-completion"), ("TTS", "llama-tts")]: + self.assertEqual(f"/tmp/canary with spaces/native/bin/{executable}", values[f"SKIPPY_WORKLOAD_ORACLE_{suffix}"]) + self.assertEqual("/tmp/canary with spaces/cargo/debug", values["SKIPPY_WORKLOAD_CANDIDATE_BIN_DIR"]) + + def test_rejects_relative_or_environment_injection_paths(self) -> None: + """Only absolute, safely exportable producer paths can enter the generated environment.""" + for path in ["relative", "/tmp/line\nGH_TOKEN=bad", "/tmp/line\rnext"]: + result = subprocess.run(["bash", str(PRODUCER), "--print-env", path], capture_output=True, check=False) + self.assertNotEqual(0, result.returncode) + + def test_both_canary_paths_build_then_export_the_same_cpu_producers(self) -> None: + """Normal and independent verification must consume the same explicit producer graph.""" + workflow = (ROOT / ".github/workflows/llama-upstream-canary.yml").read_text() + repair = (ROOT / "scripts/llama-canary-agent-repair.sh").read_text() + for text in [workflow, repair]: + self.assertIn("just skippy-workload-oracles-build", text) + self.assertIn("skippy-workload-oracles-build.sh --print-env", text) + self.assertLess(workflow.index("name: Build pinned CPU workload"), workflow.index("name: Supported-families certification battery")) + producer = PRODUCER.read_text() + self.assertIn("-p skippy-topology --bins", producer) + for contract in ["LLAMA_STAGE_BACKEND=cpu", "LLAMA_STAGE_LINK_MODE=static", "LLAMA_STAGE_WORKLOAD_ORACLE=ON", "CARGO_TARGET_DIR=", "--no-run --message-format=json", "--write-producer", "--source-snapshot"]: + self.assertIn(contract, producer) + consumer = (ROOT / "scripts/skippy-workload-certify.sh").read_text() + self.assertIn("--producer-manifest", consumer) + self.assertIn('TEST_COMMAND=("$(jq -er', consumer) + + def test_full_roster_and_cpu_producer_include_non_chat_on_every_trigger(self) -> None: + """Nightly, pin changes, and forced runs share all six certified workload classes.""" + result = subprocess.run( + [str(ROOT / "scripts/plan-family-battery.py")], + cwd=ROOT, text=True, capture_output=True, check=True, + ) + rows = [row for row in json.loads(result.stdout)["selected_models"] if row["class"] != "causal_generation"] + self.assertEqual(6, len(rows)) + self.assertEqual({"embedding", "rerank", "encoder_decoder", "ocr", "speech_synthesis", "speech_recognition"}, {row["class"] for row in rows}) + self.assertTrue(all(row["profile"] == "workload-oracle" for row in rows)) + workflow = (ROOT / ".github/workflows/llama-upstream-canary.yml").read_text() + producer_step = workflow.split("- name: Build pinned CPU workload oracles and candidate\n", 1)[1].split(" - name:", 1)[0] + self.assertNotIn("cadence", producer_step) + self.assertIn("steps.sha.outputs.certify == 'true'", producer_step) diff --git a/scripts/tests/test_static_abi_artifacts.py b/scripts/tests/test_static_abi_artifacts.py index a7bf5a0696..a694e9db70 100644 --- a/scripts/tests/test_static_abi_artifacts.py +++ b/scripts/tests/test_static_abi_artifacts.py @@ -112,6 +112,7 @@ def test_skippy_ffi_links_mtmd_hash_dependency_after_mtmd(self) -> None: ) def test_skippy_ffi_uses_the_native_build_scripts_canonical_directory(self) -> None: + """Rust FFI discovery must resolve the same backend and linkage directory as native preparation.""" ffi_build = (ROOT / "crates" / "skippy-ffi" / "build.rs").read_text( encoding="utf-8", ) @@ -123,7 +124,9 @@ def test_skippy_ffi_uses_the_native_build_scripts_canonical_directory(self) -> N for backend in ("cpu", "metal", "cuda", "rocm"): environment = os.environ.copy() + environment.pop("LLAMA_STAGE_BUILD_DIR", None) environment["LLAMA_STAGE_BACKEND"] = backend + environment["LLAMA_STAGE_LINK_MODE"] = "static" result = subprocess.run( ["bash", str(ROOT / "scripts" / "build-llama.sh"), "--print-build-dir"], cwd=ROOT, diff --git a/scripts/tests/test_verify_skippy_family_generator_coverage.py b/scripts/tests/test_verify_skippy_family_generator_coverage.py index 1345b39482..25449404db 100644 --- a/scripts/tests/test_verify_skippy_family_generator_coverage.py +++ b/scripts/tests/test_verify_skippy_family_generator_coverage.py @@ -2,10 +2,9 @@ import importlib.util import json -from pathlib import Path import tempfile import unittest - +from pathlib import Path ROOT = Path(__file__).resolve().parents[2] SCRIPT = ROOT / "scripts" / "verify-skippy-family-generator-coverage.py" @@ -34,7 +33,8 @@ def write(self, name: str, value: object) -> Path: return path def test_primary_family_requires_partitioned_decoder_transform(self) -> None: - manifest = self.write("manifest.json", {"models": [{"family": "alpha"}]}) + """Auxiliary sidecar support cannot substitute for a causal partitioned-decoder transform.""" + manifest = self.write("manifest.json", {"models": [{"family": "alpha", "class": "causal_generation", "profile": "full"}]}) family_map = self.write("map.json", {"families": {"alpha": ["src/models/alpha.cpp"]}}) report = self.write("report.json", {"builders": [{ "file": "src/models/alpha.cpp", @@ -47,7 +47,8 @@ def test_primary_family_requires_partitioned_decoder_transform(self) -> None: ) def test_mapped_partitioned_decoder_passes(self) -> None: - manifest = self.write("manifest.json", {"models": [{"family": "alpha"}]}) + """Accept a family only when its mapped source has partitioned-decoder transformation proof.""" + manifest = self.write("manifest.json", {"models": [{"family": "alpha", "class": "causal_generation", "profile": "full"}]}) family_map = self.write("map.json", {"families": {"alpha": ["src/models/alpha.cpp"]}}) report = self.write("report.json", {"builders": [{ "file": "src/models/alpha.cpp", @@ -56,6 +57,28 @@ def test_mapped_partitioned_decoder_passes(self) -> None: }]}) self.assertEqual([], self.module.verify(manifest, family_map, report)) + def test_non_chat_rows_require_workload_profiles_not_decoder_transforms(self) -> None: + """Full-model workload evidence cannot be confused with staged decoder proof.""" + family_map = self.write("map.json", {"families": {}}) + report = self.write("report.json", {"builders": []}) + for model_class in sorted(self.module.NON_CHAT_CLASSES): + for profile in ("workload-smoke", "workload-oracle"): + with self.subTest(model_class=model_class, profile=profile): + manifest = self.write("manifest.json", {"models": [{ + "family": "non-chat", "class": model_class, "profile": profile, + }]}) + self.assertEqual([], self.module.verify(manifest, family_map, report)) + + def test_missing_unknown_and_misclassified_rows_fail_closed(self) -> None: + """Changing a class or profile cannot silently bypass generator coverage.""" + family_map = self.write("map.json", {"families": {}}) + report = self.write("report.json", {"builders": []}) + for fields in ({}, {"class": "future"}, {"class": "embedding", "profile": "full"}, + {"class": "causal_generation", "profile": "workload-oracle"}): + with self.subTest(fields=fields): + manifest = self.write("manifest.json", {"models": [{"family": "alpha", **fields}]}) + self.assertEqual(1, len(self.module.verify(manifest, family_map, report))) + if __name__ == "__main__": unittest.main() diff --git a/scripts/tests/test_verify_workload_oracle_evidence.py b/scripts/tests/test_verify_workload_oracle_evidence.py new file mode 100644 index 0000000000..f9c7ad794e --- /dev/null +++ b/scripts/tests/test_verify_workload_oracle_evidence.py @@ -0,0 +1,204 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +VERIFIER = ROOT / "scripts" / "verify-workload-oracle-evidence.py" +WRITER = ROOT / "scripts" / "write-workload-oracle-evidence.py" + + +def sha256(path: Path) -> str: + """Hash fixture bytes using the same content identity recorded in evidence.""" + return hashlib.sha256(path.read_bytes()).hexdigest() + + +class WorkloadOracleEvidenceTests(unittest.TestCase): + def setUp(self) -> None: + """Create isolated model, producer and comparison evidence for each verifier case.""" + self.temp_dir = tempfile.TemporaryDirectory() + self.addCleanup(self.temp_dir.cleanup) + root = Path(self.temp_dir.name) + self.model = root / "model.gguf" + self.model.write_bytes(b"pinned-model") + self.candidate = root / "skippy-server" + self.candidate.write_bytes(b"candidate") + self.oracle = root / "llama-server" + self.oracle.write_bytes(b"monolithic") + self.evidence = root / "workload-oracle-evidence.json" + self.body = { + "status": "pass", + "class": "embedding", + "smoke_lane": "embedding-smoke", + "oracle_lane": "embedding-oracle", + "model_id": "fixture", + "model_sha256": sha256(self.model), + "projector_sha256": None, + "candidate_executable_sha256": sha256(self.candidate), + "oracle_executable": "llama-server", + "oracle_executable_sha256": sha256(self.oracle), + "pinned_patch_sha": "a" * 40, + "comparison": "embedding local-monolithic oracle passed: max_abs_delta=0, min_cosine=1", + } + + def run_verifier(self, model_class: str = "embedding", *extra: str) -> subprocess.CompletedProcess[str]: + """Invoke the verifier against the selected class and independent fixture paths.""" + return subprocess.run( + [ + "python3", str(VERIFIER), "--evidence", str(self.evidence), + "--class", model_class, "--smoke-lane", "embedding-smoke", + "--oracle-lane", "embedding-oracle", "--model-id", "fixture", + "--model-path", str(self.model), + "--candidate-executable", str(self.candidate), + "--oracle-executable", str(self.oracle), + "--pinned-patch-sha", "a" * 40, + *extra, + ], + cwd=ROOT, text=True, capture_output=True, check=False, + ) + + def run_writer(self, comparison: str, lane: str = "embedding-smoke") -> subprocess.CompletedProcess[str]: + """Run the evidence writer with a supplied comparison transcript and lane label.""" + comparison_log = Path(self.temp_dir.name) / "comparison.txt" + comparison_log.write_text(comparison + "\n", encoding="utf-8") + return subprocess.run( + [ + "python3", str(WRITER), "--output", str(self.evidence), + "--comparison-log", str(comparison_log), "--class", "embedding", + "--smoke-lane", lane, "--model-id", "fixture", + "--model-sha256", sha256(self.model), + "--candidate-executable", str(self.candidate), + "--oracle-executable", str(self.oracle), + "--pinned-patch-sha", "a" * 40, + "--work-dir", self.temp_dir.name, + ], + cwd=ROOT, text=True, capture_output=True, check=False, + ) + + def test_comparator_pass_writes_verifiable_identity_bound_evidence(self) -> None: + """A genuine comparator success produces evidence that passes independent verification.""" + written = self.run_writer(self.body["comparison"]) + self.assertEqual(0, written.returncode, written.stderr) + self.assertEqual(0, self.run_verifier().returncode) + + def test_smoke_only_log_never_writes_oracle_evidence(self) -> None: + """Protocol smoke output alone cannot be promoted to equivalence evidence.""" + written = self.run_writer("embedding OpenAI HTTP smoke passed") + self.assertEqual(1, written.returncode) + self.assertFalse(self.evidence.exists()) + + def test_writer_rejects_missing_suffix_and_replaces_only_final_suffix(self) -> None: + """Normalize only the terminal smoke suffix without changing embedded lane text.""" + for lane in ("embedding", "embedding-smoke-extra", "embedding-oracle"): + with self.subTest(lane=lane): + result = self.run_writer(self.body["comparison"], lane) + self.assertEqual(1, result.returncode) + self.assertIn("must end with '-smoke'", result.stderr) + self.assertFalse(self.evidence.exists()) + result = self.run_writer(self.body["comparison"], "fixture-smoke-embedding-smoke") + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual("fixture-smoke-embedding-oracle", json.loads(self.evidence.read_text())["oracle_lane"]) + + def test_projector_classes_require_independently_supplied_projector(self) -> None: + """Projector-bearing workloads need an external sidecar identity to verify against.""" + for model_class in ("ocr", "speech_synthesis", "speech_recognition"): + with self.subTest(model_class=model_class): + self.body["class"] = model_class + self.evidence.write_text(json.dumps(self.body), encoding="utf-8") + result = self.run_verifier(model_class) + self.assertEqual(1, result.returncode) + self.assertIn("requires a projector path", result.stderr) + + def test_projector_digest_is_verified_against_local_bytes(self) -> None: + """Modified projector bytes must invalidate otherwise matching evidence.""" + projector = Path(self.temp_dir.name) / "projector.gguf" + projector.write_bytes(b"projector") + self.body.update({"class": "ocr", "projector_sha256": sha256(projector), + "comparison": "ocr local-monolithic oracle passed: exact text"}) + self.evidence.write_text(json.dumps(self.body), encoding="utf-8") + result = self.run_verifier("ocr", "--projector-path", str(projector)) + self.assertEqual(0, result.returncode, result.stderr) + projector.write_bytes(b"different projector") + result = self.run_verifier("ocr", "--projector-path", str(projector)) + self.assertEqual(1, result.returncode) + self.assertIn("projector_sha256 does not match", result.stderr) + + def test_matching_explicit_evidence_is_accepted(self) -> None: + """Consistent explicit evidence remains accepted by the fail-closed verifier.""" + self.evidence.write_text(json.dumps(self.body), encoding="utf-8") + result = self.run_verifier() + self.assertEqual(0, result.returncode, result.stderr) + + def test_missing_evidence_is_rejected(self) -> None: + """An absent evidence document cannot certify a completed-looking workload lane.""" + result = self.run_verifier() + self.assertEqual(1, result.returncode) + self.assertIn("workload oracle evidence rejected", result.stderr) + + def test_tampered_model_identity_is_rejected(self) -> None: + """Changing the recorded model identity invalidates reference equivalence.""" + self.body["model_sha256"] = "b" * 64 + self.evidence.write_text(json.dumps(self.body), encoding="utf-8") + result = self.run_verifier() + self.assertEqual(1, result.returncode) + self.assertIn("model_sha256 does not match", result.stderr) + + def test_smoke_only_output_cannot_certify_oracle_lane(self) -> None: + """Reject evidence whose purported comparison is only a smoke transcript.""" + self.body["comparison"] = "embedding OpenAI HTTP smoke passed" + self.evidence.write_text(json.dumps(self.body), encoding="utf-8") + result = self.run_verifier() + self.assertEqual(1, result.returncode) + self.assertIn("lacks an explicit comparator pass", result.stderr) + + def test_writer_rejects_lane_without_smoke_suffix(self) -> None: + """Reject malformed lane names before an oracle artifact can be written.""" + comparison_log = Path(self.temp_dir.name) / "comparison.txt" + comparison_log.write_text(self.body["comparison"] + "\n", encoding="utf-8") + result = subprocess.run( + [ + "python3", str(WRITER), "--output", str(self.evidence), + "--comparison-log", str(comparison_log), "--class", "embedding", + "--smoke-lane", "embedding-smoke-extra", "--model-id", "fixture", + "--model-sha256", sha256(self.model), + "--candidate-executable", str(self.candidate), + "--oracle-executable", str(self.oracle), + "--pinned-patch-sha", "a" * 40, "--work-dir", self.temp_dir.name, + ], + cwd=ROOT, text=True, capture_output=True, check=False, + ) + self.assertEqual(1, result.returncode) + self.assertIn("must end with '-smoke'", result.stderr) + + def test_projector_workload_requires_projector_identity(self) -> None: + """A projector class cannot claim complete identity with only the main GGUF.""" + self.body.update({ + "class": "ocr", + "smoke_lane": "ocr-smoke", + "oracle_lane": "ocr-oracle", + "comparison": "ocr local-monolithic oracle passed: exact text", + }) + self.evidence.write_text(json.dumps(self.body), encoding="utf-8") + result = subprocess.run( + [ + "python3", str(VERIFIER), "--evidence", str(self.evidence), + "--class", "ocr", "--smoke-lane", "ocr-smoke", + "--oracle-lane", "ocr-oracle", "--model-id", "fixture", + "--model-path", str(self.model), + "--candidate-executable", str(self.candidate), + "--oracle-executable", str(self.oracle), + "--pinned-patch-sha", "a" * 40, + ], + cwd=ROOT, text=True, capture_output=True, check=False, + ) + self.assertEqual(1, result.returncode) + self.assertIn("requires a projector path", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_workload_lane_execution.py b/scripts/tests/test_workload_lane_execution.py new file mode 100644 index 0000000000..a179335967 --- /dev/null +++ b/scripts/tests/test_workload_lane_execution.py @@ -0,0 +1,137 @@ +"""Exercise the real shell lane functions without loading models or building.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import subprocess +import tempfile +import time +import unittest + +ROOT = Path(__file__).resolve().parents[2] + + +def shell_function(script: str, name: str) -> str: + """Load one top-level function verbatim, excluding the script's entrypoint.""" + source = (ROOT / "scripts" / script).read_text(encoding="utf-8") + start = source.index(f"{name}() {{\n") + end = source.index("\n}\n", start) + 3 + return source[start:end] + + +class WorkloadLaneExecutionTests(unittest.TestCase): + def test_summary_preserves_preflight_classes_without_claiming_certification(self) -> None: + """Use planned classes for preflight; environment checks have no model class.""" + classes = ["causal_generation", "embedding", "rerank", "encoder_decoder", + "ocr", "speech_synthesis", "speech_recognition"] + models = [{"family": f"family-{index}", "class": value} + for index, value in enumerate(classes)] + outcome = {"name": "model-preflight", "status": "pass", "outcome": "pass", "exit_code": 0} + rows = [{"family": model["family"], "outcomes": [outcome]} for model in models] + rows.extend([ + {"family": "battery", "outcomes": [{**outcome, "name": "environment-preflight"}]}, + {"family": "family-0", "split_layer": 2, "outcomes": [{**outcome, "name": "chain"}]}, + {"family": "explicit", "workload_class": "embedding", + "outcomes": [{**outcome, "name": "embedding-oracle"}]}, + ]) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + policy, results, summary = (root / name for name in ("plan.json", "results.jsonl", "summary.tsv")) + policy.write_text(json.dumps({"selected_models": models})) + source = "\n".join(json.dumps(row) for row in rows) + results.write_text(source) + env = {**os.environ, "POLICY_PLAN_COPY": str(policy), + "RESULTS_JSONL": str(results), "SUMMARY_TSV": str(summary)} + script = "set -euo pipefail\n" + shell_function("skippy-family-battery.sh", "write_lane_summary") + result = subprocess.run(["bash", "-c", script + "\nwrite_lane_summary"], env=env, + capture_output=True, text=True, check=False, timeout=15) + self.assertEqual(0, result.returncode, result.stderr) + actual = [line.split("\t") for line in summary.read_text().splitlines()] + self.assertEqual(["family", "class", "split_layer", "lane", "status", "outcome", "exit_code"], actual[0]) + self.assertEqual(classes + ["", "causal_generation", "embedding"], [row[1] for row in actual[1:]]) + self.assertEqual(source, results.read_text(), "summary must not promote preflight rows to certifications") + + def run_lane(self, dry_run: bool) -> tuple[subprocess.CompletedProcess[str], list[dict]]: + """Run an isolated battery function with deterministic producer and log fixtures.""" + with tempfile.TemporaryDirectory() as directory: + env = {key: value for key, value in os.environ.items() + if not key.startswith("SKIPPY_WORKLOAD_ORACLE_")} + env.update({"ROOT": str(ROOT), "CERT_DIR": directory, + "RESULTS_JSONL": str(Path(directory) / "results.jsonl"), + "DRY_RUN": str(int(dry_run))}) + script = "\n".join([ + "set -euo pipefail", "TOTAL=0; CERT_FAILURE_COUNT=0; FAILURES=()", + shell_function("skippy-family-battery.sh", "slugify"), + "cert_timeout_for_startup() { printf 1800; }", + shell_function("skippy-family-battery.sh", "run_workload_certify"), + 'run_workload_certify first embedding /unused/model.gguf fixture rev 600 1024 embedding-smoke,embedding-oracle ""', + 'run_workload_certify second rerank /unused/model.gguf fixture rev 900 1024 rerank-smoke,rerank-oracle ""', + 'printf "counts=%s,%s\\n" "$TOTAL" "$CERT_FAILURE_COUNT"', + ]) + result = subprocess.run(["bash", "-c", script], env=env, cwd=ROOT, + capture_output=True, text=True, check=False, timeout=15) + path = Path(env["RESULTS_JSONL"]) + # jq emits pretty-printed objects separated by whitespace. + rows = [] + remaining = path.read_text() if path.exists() else "" + decoder = json.JSONDecoder() + while remaining.strip(): + row, end = decoder.raw_decode(remaining.lstrip()) + rows.append(row) + remaining = remaining.lstrip()[end:] + return result, rows + + def test_dry_run_prints_both_certified_rows_without_oracles(self) -> None: + """Planning must remain usable before oracle executables are provisioned.""" + result, rows = self.run_lane(True) + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn("--startup-timeout-secs 600", result.stdout) + self.assertIn("--startup-timeout-secs 900", result.stdout) + self.assertEqual(2, result.stdout.count("--require-oracle")) + self.assertIn("counts=2,0", result.stdout) + self.assertEqual([], rows) + + def test_missing_oracle_records_both_failures_and_continues(self) -> None: + """A missing prerequisite must retain failures for every selected family.""" + result, rows = self.run_lane(False) + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn("counts=2,2", result.stdout) + self.assertEqual(["first", "second"], [row["family"] for row in rows]) + for row in rows: + self.assertEqual(1, row["exit_code"]) + self.assertEqual(2, len(row["outcomes"])) + self.assertTrue(all(lane["status"] == "fail" for lane in row["outcomes"])) + + def test_readiness_uses_deadline_for_each_server_and_rejects_dead_process(self) -> None: + """Give each server its planned budget while detecting an exited child promptly.""" + function = shell_function("skippy-workload-certify.sh", "wait_for_workload_server") + for label in ("OpenAI server", "monolithic oracle server"): + with self.subTest(label=label), tempfile.TemporaryDirectory() as directory: + log = Path(directory) / "server.log" + log.write_text("fixture startup log\n") + env = {**os.environ, "STARTUP_TIMEOUT_SECS": "1", "MODEL_CLASS": "embedding", + "MODEL_ID": "fixture", "SERVER_LOG": str(log), "LABEL": label} + prefix = "set -euo pipefail\n" + function + "\n" + start = time.monotonic() + result = subprocess.run(["bash", "-c", prefix + + 'curl() { return 1; }; wait_for_workload_server $$ 1 "$SERVER_LOG" "$LABEL"'], + env=env, capture_output=True, text=True, check=False, timeout=5) + self.assertEqual(1, result.returncode) + self.assertLess(time.monotonic() - start, 3) + self.assertIn(f"{label} was not ready within 1 seconds", result.stderr) + self.assertIn("fixture startup log", result.stderr) + result = subprocess.run(["bash", "-c", prefix + + 'wait_for_workload_server 99999999 1 "$SERVER_LOG" "$LABEL"'], + env=env, capture_output=True, text=True, check=False, timeout=5) + self.assertEqual(1, result.returncode) + self.assertIn(f"{label} exited early", result.stderr) + result = subprocess.run(["bash", "-c", prefix + + 'curl() { printf \'{"data":[{"id":"fixture"}]}\'; }; wait_for_workload_server $$ 1 "$SERVER_LOG" "$LABEL"'], + env=env, capture_output=True, text=True, check=False, timeout=5) + self.assertEqual(0, result.returncode, result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_workload_monolithic_oracle.py b/scripts/tests/test_workload_monolithic_oracle.py new file mode 100644 index 0000000000..5fa47f33b7 --- /dev/null +++ b/scripts/tests/test_workload_monolithic_oracle.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path +import subprocess +import sys +import unittest +from unittest.mock import patch + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts" / "ci-workload-monolithic-oracle.py" +sys.path.insert(0, str(SCRIPT.parent)) +SPEC = importlib.util.spec_from_file_location("workload_monolithic_oracle", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +oracle = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(oracle) + + +class WorkloadMonolithicOracleTests(unittest.TestCase): + def test_embedding_requires_dimension_and_numeric_parity(self) -> None: + """Require matching dimensions and bounded component error, not merely valid vectors.""" + reference = { + "data": [ + {"index": index, "embedding": [1.0, 0.0]} + for index in range(len(oracle.EMBEDDING_INPUTS)) + ] + } + self.assertIn("max_abs_delta=0", oracle.compare_embeddings(reference, reference)) + changed = { + "data": [ + {"index": index, "embedding": [0.9, 0.1]} + for index in range(len(oracle.EMBEDDING_INPUTS)) + ] + } + with self.assertRaisesRegex(RuntimeError, "differs from monolithic reference"): + oracle.compare_embeddings(changed, reference) + changed["data"][0]["embedding"] = [1.0] + with self.assertRaisesRegex(RuntimeError, "dimensions differ"): + oracle.compare_embeddings(changed, reference) + + def test_embedding_oracle_checks_each_single_input_after_batch(self) -> None: + """Batch parity cannot hide divergence when the same inputs execute individually.""" + batch = {"data": [ + {"index": index, "embedding": [1.0, 0.0]} + for index in range(len(oracle.EMBEDDING_INPUTS)) + ]} + single = {"data": [{"index": 0, "embedding": [1.0, 0.0]}]} + divergent = {"data": [{"index": 0, "embedding": [0.0, 1.0]}]} + responses = [batch, batch, single, single, single, divergent, single, single] + with patch.object(oracle, "request_json", side_effect=responses) as request: + with self.assertRaisesRegex(RuntimeError, r"single\[1\]"): + oracle.run_embedding_oracle("http://candidate", "http://oracle", "fixture") + self.assertEqual(8, request.call_count) + self.assertEqual(oracle.EMBEDDING_INPUTS[0], request.call_args_list[2].args[2]["input"]) + + def test_rerank_requires_same_scores_and_order(self) -> None: + """A changed ranking or relevance score must fail reference equivalence.""" + reference = {"results": [ + {"index": 0, "relevance_score": 2.0}, + {"index": 1, "relevance_score": -1.0}, + ]} + self.assertIn("max_abs_delta=0", oracle.compare_rerank(reference, reference)) + changed = {"results": [ + {"index": 0, "relevance_score": 0.1}, + {"index": 1, "relevance_score": 0.2}, + ]} + with self.assertRaisesRegex(RuntimeError, "differs from monolithic reference"): + oracle.compare_rerank(changed, reference) + + def test_encoder_decoder_compares_normalized_text(self) -> None: + """Ignore whitespace decoration but reject a semantically different completion.""" + reference = {"choices": [{"text": "Das Haus ist wunderbar."}]} + equivalent = {"choices": [{"text": " Das Haus ist wunderbar.\n"}]} + self.assertIn("identical normalized text", oracle.compare_encoder_decoder(equivalent, reference)) + changed = {"choices": [{"text": "Das Auto ist wunderbar."}]} + with self.assertRaisesRegex(RuntimeError, "differs from monolithic reference"): + oracle.compare_encoder_decoder(changed, reference) + + def test_direct_monolithic_completion_strips_only_terminal_runner_marker(self) -> None: + """Remove the CLI terminator without admitting displayed prompts as generated text.""" + result = subprocess.CompletedProcess( + args=["llama-completion"], returncode=0, + stdout=" Das Haus ist schön. [end of text]\n", stderr="model loaded", + ) + with patch.object(oracle.subprocess, "run", return_value=result) as run: + response = oracle.monolithic_completion("/bin/llama-completion", "/model.gguf") + self.assertEqual("Das Haus ist schön.", response["choices"][0]["text"]) + self.assertIn("--no-display-prompt", run.call_args.args[0]) + self.assertIn("--temp", run.call_args.args[0]) + + def test_direct_monolithic_completion_rejects_empty_or_failed_output(self) -> None: + """A successful process with no generated text is not passing oracle evidence.""" + empty = subprocess.CompletedProcess(args=[], returncode=0, + stdout=" [end of text]\n", stderr="") + with patch.object(oracle.subprocess, "run", return_value=empty): + with self.assertRaisesRegex(RuntimeError, "produced no text"): + oracle.monolithic_completion("/bin/llama-completion", "/model.gguf") + failed = subprocess.CompletedProcess(args=[], returncode=1, stdout="", stderr="bad") + with patch.object(oracle.subprocess, "run", return_value=failed): + with self.assertRaisesRegex(RuntimeError, "exited 1"): + oracle.monolithic_completion("/bin/llama-completion", "/model.gguf") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/verify-skippy-family-generator-coverage.py b/scripts/verify-skippy-family-generator-coverage.py index ff2629afe3..a6d2a15d74 100755 --- a/scripts/verify-skippy-family-generator-coverage.py +++ b/scripts/verify-skippy-family-generator-coverage.py @@ -7,8 +7,14 @@ import json from pathlib import Path +NON_CHAT_CLASSES = { + "embedding", "rerank", "encoder_decoder", "ocr", + "speech_synthesis", "speech_recognition", +} + def verify(manifest_path: Path, family_map_path: Path, report_path: Path) -> list[str]: + """Require decoder transforms only for explicitly classified causal split targets.""" manifest = json.loads(manifest_path.read_text(encoding="utf-8")) family_map = json.loads(family_map_path.read_text(encoding="utf-8"))["families"] report = json.loads(report_path.read_text(encoding="utf-8")) @@ -21,6 +27,17 @@ def verify(manifest_path: Path, family_map_path: Path, report_path: Path) -> lis errors: list[str] = [] for model in manifest.get("models", []): family = model.get("family", "") + model_class = model.get("class") + if isinstance(model_class, str) and model_class in NON_CHAT_CLASSES: + if model.get("profile") not in ("workload-smoke", "workload-oracle"): + errors.append(f"{family}: non-chat class requires a workload profile") + continue + if model_class != "causal_generation": + errors.append(f"{family}: missing or unknown workload class") + continue + if model.get("profile") in ("workload-smoke", "workload-oracle"): + errors.append(f"{family}: causal split target cannot use a workload profile") + continue sources = family_map.get(family) if not sources: errors.append(f"{family}: no generated family source mapping") @@ -34,6 +51,7 @@ def verify(manifest_path: Path, family_map_path: Path, report_path: Path) -> lis def main() -> int: + """Check causal decoder coverage without promoting full-model workload evidence.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--manifest", type=Path, required=True) parser.add_argument("--family-map", type=Path, required=True) @@ -42,7 +60,7 @@ def main() -> int: errors = verify(args.manifest, args.family_map, args.report) if errors: raise SystemExit("generator coverage failed:\n " + "\n ".join(errors)) - print("every primary canary family owns a transformed partitioned decoder") + print("every causal canary split target owns a transformed partitioned decoder") return 0 diff --git a/scripts/verify-workload-oracle-evidence.py b/scripts/verify-workload-oracle-evidence.py new file mode 100644 index 0000000000..dedf10e72e --- /dev/null +++ b/scripts/verify-workload-oracle-evidence.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Verify an observed non-chat oracle result before battery certification.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import sys + + +ORACLE_EXECUTABLE = { + "embedding": "llama-server", + "rerank": "llama-server", + "encoder_decoder": "llama-completion", + "ocr": "llama-server", + "speech_synthesis": "llama-tts", + "speech_recognition": "llama-server", +} + + +def sha256(path: Path) -> str: + """Hash independently supplied artifact bytes for comparison with evidence.""" + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def verify(args: argparse.Namespace) -> None: + """Reject missing prerequisites or evidence not bound to these exact inputs.""" + if args.model_class in {"ocr", "speech_synthesis", "speech_recognition"} and not args.projector_path: + raise ValueError(f"{args.model_class} oracle evidence requires a projector path") + evidence = json.loads(args.evidence.read_text(encoding="utf-8")) + if not isinstance(evidence, dict): + raise ValueError("oracle evidence must be an object") + expected = { + "status": "pass", + "class": args.model_class, + "smoke_lane": args.smoke_lane, + "oracle_lane": args.oracle_lane, + "model_id": args.model_id, + "model_sha256": sha256(args.model_path), + "projector_sha256": sha256(args.projector_path) if args.projector_path else None, + "oracle_executable": ORACLE_EXECUTABLE[args.model_class], + "oracle_executable_sha256": sha256(args.oracle_executable), + "candidate_executable_sha256": sha256(args.candidate_executable), + "pinned_patch_sha": args.pinned_patch_sha, + } + for key, value in expected.items(): + if evidence.get(key) != value: + raise ValueError(f"oracle evidence {key} does not match this run") + if args.oracle_executable.name != expected["oracle_executable"]: + raise ValueError("wrong oracle executable for workload class") + comparison = evidence.get("comparison") + if not isinstance(comparison, str) or not comparison.startswith( + f"{args.model_class} local-monolithic oracle passed: " + ): + raise ValueError("oracle evidence lacks an explicit comparator pass") + if args.model_class == "speech_synthesis" and not isinstance(evidence.get("metrics"), dict): + raise ValueError("TTS oracle evidence lacks PCM metrics") + + +def main() -> int: + """Return success only when the recorded comparator pass matches this run.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--evidence", required=True, type=Path) + parser.add_argument("--class", dest="model_class", required=True, choices=ORACLE_EXECUTABLE) + parser.add_argument("--smoke-lane", required=True) + parser.add_argument("--oracle-lane", required=True) + parser.add_argument("--model-id", required=True) + parser.add_argument("--model-path", required=True, type=Path) + parser.add_argument("--projector-path", type=Path) + parser.add_argument("--candidate-executable", required=True, type=Path) + parser.add_argument("--oracle-executable", required=True, type=Path) + parser.add_argument("--pinned-patch-sha", required=True) + args = parser.parse_args() + try: + verify(args) + except (OSError, ValueError, KeyError, json.JSONDecodeError) as error: + print(f"workload oracle evidence rejected: {error}", file=sys.stderr) + return 1 + print(f"verified {args.model_class} local-monolithic oracle evidence") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/workload_fixtures.py b/scripts/workload_fixtures.py new file mode 100644 index 0000000000..500511261f --- /dev/null +++ b/scripts/workload_fixtures.py @@ -0,0 +1,15 @@ +"""Shared deterministic prompts for non-chat smoke and reference comparisons.""" + +EMBEDDING_INPUTS = ( + "search_query: distributed GPU inference", + "search_document: GPUs share one language model over a mesh", + "search_document: A recipe for tomato soup", +) + +RERANK_QUERY = "distributed GPU inference" +RERANK_DOCUMENTS = ( + "GPUs share one language model over a mesh", + "A recipe for tomato soup", +) + +ENCODER_DECODER_PROMPT = "translate English to German: The house is wonderful." diff --git a/scripts/write-workload-oracle-evidence.py b/scripts/write-workload-oracle-evidence.py new file mode 100644 index 0000000000..ee24a7c665 --- /dev/null +++ b/scripts/write-workload-oracle-evidence.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Persist an explicit, identity-bound workload oracle pass for the battery.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import sys + + +def sha256(path: Path) -> str: + """Hash a model sidecar or executable without buffering the whole artifact.""" + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def write_evidence(args: argparse.Namespace) -> None: + """Bind an observed comparator pass to its lane and executable identities.""" + if not args.smoke_lane.endswith("-smoke"): + raise ValueError(f"smoke lane must end with '-smoke': {args.smoke_lane}") + lines = [ + line.strip() + for line in args.comparison_log.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + if not lines or not lines[-1].startswith( + f"{args.model_class} local-monolithic oracle passed: " + ): + raise ValueError("oracle comparator did not emit an explicit class-specific pass") + evidence: dict[str, object] = { + "status": "pass", + "class": args.model_class, + "smoke_lane": args.smoke_lane, + "oracle_lane": f"{args.smoke_lane.removesuffix('-smoke')}-oracle", + "model_id": args.model_id, + "model_sha256": args.model_sha256, + "projector_sha256": sha256(args.projector_path) if args.projector_path else None, + "oracle_executable": args.oracle_executable.name, + "oracle_executable_sha256": sha256(args.oracle_executable), + "candidate_executable_sha256": sha256(args.candidate_executable), + "pinned_patch_sha": args.pinned_patch_sha, + "comparison": lines[-1], + } + if args.model_class == "speech_synthesis": + result = json.loads((args.work_dir / "tts-oracle-result.json").read_text(encoding="utf-8")) + if result.get("status") != "pass" or result.get("pinned_patch_sha") != args.pinned_patch_sha: + raise ValueError("TTS comparator result is missing or does not match the pinned patch") + if not isinstance(result.get("metrics"), dict): + raise ValueError("TTS comparator result lacks PCM metrics") + evidence["metrics"] = result["metrics"] + args.output.write_text(json.dumps(evidence, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def main() -> int: + """Persist a verified comparator result, returning failure for incomplete evidence.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--comparison-log", required=True, type=Path) + parser.add_argument("--class", dest="model_class", required=True) + parser.add_argument("--smoke-lane", required=True) + parser.add_argument("--model-id", required=True) + parser.add_argument("--model-sha256", required=True) + parser.add_argument("--projector-path", type=Path) + parser.add_argument("--candidate-executable", required=True, type=Path) + parser.add_argument("--oracle-executable", required=True, type=Path) + parser.add_argument("--pinned-patch-sha", required=True) + parser.add_argument("--work-dir", required=True, type=Path) + args = parser.parse_args() + try: + write_evidence(args) + except (OSError, ValueError) as error: + print(f"workload oracle evidence not written: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/third_party/llama.cpp/patches/0043-skippy-define-non-chat-workload-ABI-and-execution.patch b/third_party/llama.cpp/patches/0043-skippy-define-non-chat-workload-ABI-and-execution.patch new file mode 100644 index 0000000000..420d17fb98 --- /dev/null +++ b/third_party/llama.cpp/patches/0043-skippy-define-non-chat-workload-ABI-and-execution.patch @@ -0,0 +1,631 @@ +From e72313edc8a778db1f8d61bcd3739271e43031c3 Mon Sep 17 00:00:00 2001 +From: Mesh-LLM CI +Date: Sat, 12 Sep 2026 01:35:50 -0700 +Subject: [PATCH 34/38] skippy: define non-chat workload ABI and execution + +--- + include/skippy.h | 1 + + include/skippy/common.h | 4 +- + include/skippy/workloads.h | 87 +++++++ + src/CMakeLists.txt | 1 + + src/skippy/abi.cpp | 3 +- + src/skippy/rerank_template.h | 36 +++ + src/skippy/workloads.cpp | 356 ++++++++++++++++++++++++++ + tests/CMakeLists.txt | 1 + + tests/test-skippy-rerank-template.cpp | 34 +++ + 9 files changed, 521 insertions(+), 2 deletions(-) + create mode 100644 include/skippy/workloads.h + create mode 100644 src/skippy/rerank_template.h + create mode 100644 src/skippy/workloads.cpp + create mode 100644 tests/test-skippy-rerank-template.cpp + +diff --git a/include/skippy.h b/include/skippy.h +index 33a47b244..dc8cbb431 100644 +--- a/include/skippy.h ++++ b/include/skippy.h +@@ -24,5 +24,6 @@ + #include "skippy/tokenization.h" + #include "skippy/model_package.h" + #include "skippy/signals.h" ++#include "skippy/workloads.h" + + #endif // SKIPPY_H +diff --git a/include/skippy/common.h b/include/skippy/common.h +index 22a116414..336985cb5 100644 +--- a/include/skippy/common.h ++++ b/include/skippy/common.h +@@ -46,7 +46,7 @@ extern "C" { + + #define SKIPPY_ABI_VERSION_MAJOR 0 + #define SKIPPY_ABI_VERSION_MINOR 1 +-#define SKIPPY_ABI_VERSION_PATCH 54 ++#define SKIPPY_ABI_VERSION_PATCH 55 + + #if defined(_MSC_VER) + #define SKIPPY_DEPRECATED(message) __declspec(deprecated(message)) +@@ -104,6 +104,8 @@ enum skippy_feature { + #define SKIPPY_FEATURE_DEVICE_EVENTS ((uint64_t)1 << 34) + #define SKIPPY_FEATURE_DIAGNOSTIC_EVENTS ((uint64_t)1 << 35) + #define SKIPPY_FEATURE_UNLOAD_EVENTS ((uint64_t)1 << 36) ++/** @brief Full-model embedding, reranking, and encoder-decoder execution. */ ++#define SKIPPY_FEATURE_NON_CHAT_WORKLOADS ((uint64_t)1 << 37) + + enum skippy_status { + SKIPPY_STATUS_OK = 0, +diff --git a/include/skippy/workloads.h b/include/skippy/workloads.h +new file mode 100644 +index 000000000..0ee115741 +--- /dev/null ++++ b/include/skippy/workloads.h +@@ -0,0 +1,87 @@ ++#ifndef SKIPPY_WORKLOADS_H ++#define SKIPPY_WORKLOADS_H ++ ++/** ++ * @file skippy/workloads.h ++ * @brief Full-model embedding, reranking, and encoder-decoder execution. ++ * ++ * These operations deliberately reject filtered stage models. Their explicit ++ * workload descriptor lets callers fail closed instead of inferring support ++ * from a model name or architecture family. ++ */ ++ ++#include "common.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++struct skippy_model; ++struct skippy_session; ++ ++#define SKIPPY_WORKLOAD_INFO_V1_ABI_VERSION 1 ++ ++enum skippy_workload_kind { ++ SKIPPY_WORKLOAD_CAUSAL_GENERATION = 0, ++ SKIPPY_WORKLOAD_EMBEDDING = 1, ++ SKIPPY_WORKLOAD_RERANK = 2, ++ SKIPPY_WORKLOAD_ENCODER_DECODER = 3, ++}; ++ ++enum skippy_workload_pooling { ++ SKIPPY_WORKLOAD_POOLING_UNSPECIFIED = -1, ++ SKIPPY_WORKLOAD_POOLING_NONE = 0, ++ SKIPPY_WORKLOAD_POOLING_MEAN = 1, ++ SKIPPY_WORKLOAD_POOLING_CLS = 2, ++ SKIPPY_WORKLOAD_POOLING_LAST = 3, ++ SKIPPY_WORKLOAD_POOLING_RANK = 4, ++}; ++ ++/** @brief Runtime-probed workload properties for an opened model. */ ++struct skippy_workload_info_v1 { ++ uint32_t abi_version; ++ uint32_t struct_size; ++ enum skippy_workload_kind kind; ++ enum skippy_workload_pooling pooling; ++ uint32_t output_dimensions; ++ uint32_t classifier_outputs; ++ bool has_encoder; ++ bool has_decoder; ++ bool full_model_only; ++ uint8_t reserved0; ++}; ++ ++/** @brief Describes the workload implemented by an opened model. */ ++LLAMA_API enum skippy_status skippy_model_workload_info_v1(const struct skippy_model * model, ++ struct skippy_workload_info_v1 * out_info, ++ struct skippy_error ** out_error); ++ ++/** @brief Computes one normalized embedding for a tokenized input. */ ++LLAMA_API enum skippy_status skippy_session_embed(struct skippy_session * session, ++ const llama_token * token_ids, ++ size_t token_count, ++ float * output, ++ size_t output_capacity, ++ size_t * out_dimensions, ++ struct skippy_error ** out_error); ++ ++/** @brief Computes one scalar relevance score for a query-document pair. */ ++LLAMA_API enum skippy_status skippy_session_rerank(struct skippy_session * session, ++ const char * query, ++ const char * document, ++ float * out_score, ++ size_t * out_token_count, ++ struct skippy_error ** out_error); ++ ++/** @brief Encodes an encoder-decoder prompt and returns its first decoder token. */ ++LLAMA_API enum skippy_status skippy_session_encode_prompt(struct skippy_session * session, ++ const llama_token * token_ids, ++ size_t token_count, ++ llama_token * out_decoder_start_token, ++ struct skippy_error ** out_error); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif // SKIPPY_WORKLOADS_H +diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt +index 6e3b7b129..1d9049b4e 100644 +--- a/src/CMakeLists.txt ++++ b/src/CMakeLists.txt +@@ -49,6 +49,7 @@ add_library(llama + skippy/model_source.cpp + skippy/session.cpp + skippy/verification.cpp ++ skippy/workloads.cpp + skippy/abi.cpp + skippy/block_boundaries.cpp + skippy/devices.cpp +diff --git a/src/skippy/abi.cpp b/src/skippy/abi.cpp +index 7227dfc48..b53f4f76b 100644 +--- a/src/skippy/abi.cpp ++++ b/src/skippy/abi.cpp +@@ -45,7 +45,8 @@ uint64_t skippy_abi_features(void) { + SKIPPY_FEATURE_KV_EVENTS | + SKIPPY_FEATURE_DEVICE_EVENTS | + SKIPPY_FEATURE_DIAGNOSTIC_EVENTS | +- SKIPPY_FEATURE_UNLOAD_EVENTS; ++ SKIPPY_FEATURE_UNLOAD_EVENTS | ++ SKIPPY_FEATURE_NON_CHAT_WORKLOADS; + } + + void skippy_error_free(struct skippy_error * error) { +diff --git a/src/skippy/rerank_template.h b/src/skippy/rerank_template.h +new file mode 100644 +index 000000000..f0c3b2ea7 +--- /dev/null ++++ b/src/skippy/rerank_template.h +@@ -0,0 +1,36 @@ ++#pragma once ++ ++#include ++#include ++#include ++#include ++ ++// Substitute only markers from the model-authored template. User-supplied ++// query/document bytes are appended verbatim and must never be parsed again. ++inline std::string skippy_render_rerank_template( ++ std::string_view prompt_template, ++ std::string_view query, ++ std::string_view document) { ++ constexpr std::string_view query_marker = "{query}"; ++ constexpr std::string_view document_marker = "{document}"; ++ std::string rendered; ++ std::size_t cursor = 0; ++ while (cursor < prompt_template.size()) { ++ const std::size_t query_pos = prompt_template.find(query_marker, cursor); ++ const std::size_t document_pos = prompt_template.find(document_marker, cursor); ++ const std::size_t next = std::min(query_pos, document_pos); ++ if (next == std::string_view::npos) { ++ rendered.append(prompt_template.data() + cursor, prompt_template.size() - cursor); ++ break; ++ } ++ rendered.append(prompt_template.data() + cursor, next - cursor); ++ if (next == query_pos) { ++ rendered.append(query.data(), query.size()); ++ cursor = next + query_marker.size(); ++ } else { ++ rendered.append(document.data(), document.size()); ++ cursor = next + document_marker.size(); ++ } ++ } ++ return rendered; ++} +diff --git a/src/skippy/workloads.cpp b/src/skippy/workloads.cpp +new file mode 100644 +index 000000000..81ceb3473 +--- /dev/null ++++ b/src/skippy/workloads.cpp +@@ -0,0 +1,356 @@ ++#include "skippy/workloads.h" ++ ++#include "llama.h" ++#include "llama-model.h" ++#include "skippy/errors.h" ++#include "skippy/rerank_template.h" ++#include "skippy/runtime_support.h" ++#include "skippy/runtime_types.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++namespace { ++ ++enum skippy_workload_kind workload_kind(const skippy_model * model) { ++ const bool has_encoder = llama_model_has_encoder(model->model); ++ const bool has_decoder = llama_model_has_decoder(model->model); ++ const enum llama_pooling_type pooling = llama_pooling_type(model->ctx); ++ ++ if (model->model->cls != nullptr || model->model->cls_out != nullptr || pooling == LLAMA_POOLING_TYPE_RANK) { ++ return SKIPPY_WORKLOAD_RERANK; ++ } ++ if (has_encoder && has_decoder) { ++ return SKIPPY_WORKLOAD_ENCODER_DECODER; ++ } ++ if (has_encoder || (pooling != LLAMA_POOLING_TYPE_NONE && pooling != LLAMA_POOLING_TYPE_UNSPECIFIED)) { ++ return SKIPPY_WORKLOAD_EMBEDDING; ++ } ++ return SKIPPY_WORKLOAD_CAUSAL_GENERATION; ++} ++ ++bool is_full_model(const skippy_model * model) { ++ return model != nullptr && !model->config.filter_tensors_on_load; ++} ++ ++enum skippy_status require_workload(skippy_session * session, ++ enum skippy_workload_kind required, ++ skippy_error ** out_error) { ++ if (session == nullptr || session->stage_model == nullptr || session->stage_model->model == nullptr || ++ session->ctx == nullptr) { ++ skippy_set_error(out_error, SKIPPY_STATUS_INVALID_ARGUMENT, "an active session is required"); ++ return SKIPPY_STATUS_INVALID_ARGUMENT; ++ } ++ if (!is_full_model(session->stage_model)) { ++ skippy_set_error(out_error, SKIPPY_STATUS_UNSUPPORTED, "non-chat workloads currently require a full model"); ++ return SKIPPY_STATUS_UNSUPPORTED; ++ } ++ if (workload_kind(session->stage_model) != required) { ++ skippy_set_error(out_error, SKIPPY_STATUS_UNSUPPORTED, "model does not implement the requested workload"); ++ return SKIPPY_STATUS_UNSUPPORTED; ++ } ++ return SKIPPY_STATUS_OK; ++} ++ ++enum skippy_status validate_tokens(skippy_session * session, ++ const llama_token * token_ids, ++ size_t token_count, ++ skippy_error ** out_error) { ++ if (token_ids == nullptr || token_count == 0) { ++ skippy_set_error(out_error, SKIPPY_STATUS_INVALID_ARGUMENT, "at least one token is required"); ++ return SKIPPY_STATUS_INVALID_ARGUMENT; ++ } ++ if (token_count > static_cast(std::numeric_limits::max()) || ++ token_count > static_cast(llama_n_batch(session->ctx))) { ++ skippy_set_error(out_error, SKIPPY_STATUS_INVALID_ARGUMENT, "token count exceeds the model batch capacity"); ++ return SKIPPY_STATUS_INVALID_ARGUMENT; ++ } ++ return SKIPPY_STATUS_OK; ++} ++ ++void clear_workload_state(skippy_session * session) { ++ skippy_synchronize_context_if_pending(session->stage_model); ++ if (llama_memory_t memory = llama_get_memory(session->ctx)) { ++ llama_memory_clear(memory, true); ++ } ++ session->n_past = 0; ++ session->token_history.clear(); ++ session->signal_history.clear(); ++} ++ ++llama_batch make_embedding_batch(skippy_session * session, const llama_token * token_ids, size_t token_count) { ++ llama_batch batch = llama_batch_init(static_cast(token_count), 0, 1); ++ batch.n_tokens = static_cast(token_count); ++ for (int32_t index = 0; index < batch.n_tokens; ++index) { ++ batch.token[index] = token_ids[index]; ++ batch.pos[index] = index; ++ batch.n_seq_id[index] = 1; ++ batch.seq_id[index][0] = session->seq_id; ++ batch.logits[index] = 1; ++ } ++ return batch; ++} ++ ++enum skippy_status decode_embedding(skippy_session * session, ++ const llama_token * token_ids, ++ size_t token_count, ++ const float ** out_embedding, ++ skippy_error ** out_error) { ++ clear_workload_state(session); ++ llama_batch batch = make_embedding_batch(session, token_ids, token_count); ++ const int32_t decode_status = llama_decode(session->ctx, batch); ++ llama_batch_free(batch); ++ if (decode_status != 0) { ++ skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, "llama_decode failed for non-chat workload"); ++ return SKIPPY_STATUS_RUNTIME_ERROR; ++ } ++ skippy_mark_context_work_pending(session->stage_model); ++ ++ const enum llama_pooling_type pooling = llama_pooling_type(session->ctx); ++ const float * embedding = pooling == LLAMA_POOLING_TYPE_NONE ? ++ llama_get_embeddings_ith(session->ctx, static_cast(token_count - 1)) : ++ llama_get_embeddings_seq(session->ctx, session->seq_id); ++ skippy_mark_context_synchronized(session->stage_model); ++ if (embedding == nullptr) { ++ skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, "model produced no workload output"); ++ return SKIPPY_STATUS_RUNTIME_ERROR; ++ } ++ *out_embedding = embedding; ++ session->n_past = static_cast(token_count); ++ session->token_history.assign(token_ids, token_ids + token_count); ++ return SKIPPY_STATUS_OK; ++} ++ ++void normalize_embedding(const float * source, float * output, size_t dimensions) { ++ double squared_norm = 0.0; ++ for (size_t index = 0; index < dimensions; ++index) { ++ squared_norm += static_cast(source[index]) * source[index]; ++ } ++ const float scale = squared_norm > 0.0 ? static_cast(1.0 / std::sqrt(squared_norm)) : 1.0f; ++ for (size_t index = 0; index < dimensions; ++index) { ++ output[index] = source[index] * scale; ++ } ++} ++ ++std::string token_piece(const llama_vocab * vocab, llama_token token) { ++ char stack[128]; ++ int32_t length = llama_token_to_piece(vocab, token, stack, sizeof(stack), 0, true); ++ if (length >= 0) { ++ return std::string(stack, static_cast(length)); ++ } ++ std::string dynamic(static_cast(-length), '\0'); ++ length = llama_token_to_piece(vocab, token, dynamic.data(), static_cast(dynamic.size()), 0, true); ++ return length > 0 ? dynamic.substr(0, static_cast(length)) : std::string(); ++} ++ ++bool tokenize(const llama_vocab * vocab, const std::string & text, std::vector & tokens) { ++ if (text.size() > static_cast(std::numeric_limits::max())) { ++ return false; ++ } ++ const int32_t text_length = static_cast(text.size()); ++ int32_t count = llama_tokenize(vocab, text.data(), text_length, nullptr, 0, true, true); ++ if (count >= 0) { ++ tokens.resize(static_cast(count)); ++ } else { ++ tokens.resize(static_cast(-count)); ++ } ++ count = ++ llama_tokenize(vocab, text.data(), text_length, tokens.data(), static_cast(tokens.size()), true, true); ++ if (count < 0) { ++ return false; ++ } ++ tokens.resize(static_cast(count)); ++ return !tokens.empty(); ++} ++ ++std::string rerank_prompt(const llama_model * model, const char * query, const char * document) { ++ if (const char * prompt_template = llama_model_chat_template(model, "rerank")) { ++ return skippy_render_rerank_template(prompt_template, query, document); ++ } ++ ++ const llama_vocab * vocab = llama_model_get_vocab(model); ++ std::string prompt(query); ++ if (llama_vocab_get_add_eos(vocab)) { ++ prompt += token_piece(vocab, llama_vocab_eos(vocab)); ++ } ++ if (llama_vocab_get_add_sep(vocab)) { ++ prompt += token_piece(vocab, llama_vocab_sep(vocab)); ++ } ++ prompt += document; ++ return prompt; ++} ++ ++} // namespace ++ ++extern "C" { ++ ++enum skippy_status skippy_model_workload_info_v1(const skippy_model * model, ++ skippy_workload_info_v1 * out_info, ++ skippy_error ** out_error) { ++ if (model == nullptr || model->model == nullptr || model->ctx == nullptr || out_info == nullptr) { ++ skippy_set_error(out_error, SKIPPY_STATUS_INVALID_ARGUMENT, "model and out_info are required"); ++ return SKIPPY_STATUS_INVALID_ARGUMENT; ++ } ++ if (out_info->abi_version != SKIPPY_WORKLOAD_INFO_V1_ABI_VERSION || ++ out_info->struct_size != sizeof(skippy_workload_info_v1)) { ++ skippy_set_error(out_error, SKIPPY_STATUS_INVALID_ARGUMENT, "incompatible workload descriptor ABI"); ++ return SKIPPY_STATUS_INVALID_ARGUMENT; ++ } ++ ++ const skippy_workload_kind kind = workload_kind(model); ++ const uint32_t output_dimensions = kind == SKIPPY_WORKLOAD_EMBEDDING ? ++ static_cast(std::max(llama_model_n_embd_out(model->model), 0)) : ++ 0; ++ const uint32_t classifier_outputs = kind == SKIPPY_WORKLOAD_RERANK ? llama_model_n_cls_out(model->model) : 0; ++ *out_info = { ++ SKIPPY_WORKLOAD_INFO_V1_ABI_VERSION, ++ static_cast(sizeof(skippy_workload_info_v1)), ++ kind, ++ static_cast(llama_pooling_type(model->ctx)), ++ output_dimensions, ++ classifier_outputs, ++ llama_model_has_encoder(model->model), ++ llama_model_has_decoder(model->model), ++ true, ++ 0, ++ }; ++ return skippy_success(out_error); ++} ++ ++enum skippy_status skippy_session_embed(skippy_session * session, ++ const llama_token * token_ids, ++ size_t token_count, ++ float * output, ++ size_t output_capacity, ++ size_t * out_dimensions, ++ skippy_error ** out_error) { ++ if (out_dimensions != nullptr) { ++ *out_dimensions = 0; ++ } ++ enum skippy_status status = require_workload(session, SKIPPY_WORKLOAD_EMBEDDING, out_error); ++ if (status != SKIPPY_STATUS_OK) { ++ return status; ++ } ++ if (output == nullptr || out_dimensions == nullptr) { ++ skippy_set_error(out_error, SKIPPY_STATUS_INVALID_ARGUMENT, "output and out_dimensions are required"); ++ return SKIPPY_STATUS_INVALID_ARGUMENT; ++ } ++ status = validate_tokens(session, token_ids, token_count, out_error); ++ if (status != SKIPPY_STATUS_OK) { ++ return status; ++ } ++ const int32_t native_dimensions = llama_model_n_embd_out(session->stage_model->model); ++ if (native_dimensions <= 0) { ++ skippy_set_error(out_error, SKIPPY_STATUS_MODEL_ERROR, "model has no embedding output dimensions"); ++ return SKIPPY_STATUS_MODEL_ERROR; ++ } ++ const size_t dimensions = static_cast(native_dimensions); ++ *out_dimensions = dimensions; ++ if (output_capacity < dimensions) { ++ skippy_set_error(out_error, SKIPPY_STATUS_BUFFER_TOO_SMALL, "embedding output buffer is too small"); ++ return SKIPPY_STATUS_BUFFER_TOO_SMALL; ++ } ++ ++ const float * embedding = nullptr; ++ status = decode_embedding(session, token_ids, token_count, &embedding, out_error); ++ if (status != SKIPPY_STATUS_OK) { ++ return status; ++ } ++ normalize_embedding(embedding, output, dimensions); ++ return skippy_success(out_error); ++} ++ ++enum skippy_status skippy_session_rerank(skippy_session * session, ++ const char * query, ++ const char * document, ++ float * out_score, ++ size_t * out_token_count, ++ skippy_error ** out_error) { ++ if (out_token_count != nullptr) { ++ *out_token_count = 0; ++ } ++ enum skippy_status status = require_workload(session, SKIPPY_WORKLOAD_RERANK, out_error); ++ if (status != SKIPPY_STATUS_OK) { ++ return status; ++ } ++ if (query == nullptr || document == nullptr || out_score == nullptr || out_token_count == nullptr) { ++ skippy_set_error(out_error, SKIPPY_STATUS_INVALID_ARGUMENT, ++ "query, document, and output pointers are required"); ++ return SKIPPY_STATUS_INVALID_ARGUMENT; ++ } ++ if (llama_model_n_cls_out(session->stage_model->model) != 1) { ++ skippy_set_error(out_error, SKIPPY_STATUS_UNSUPPORTED, "rerank API requires a single classifier output"); ++ return SKIPPY_STATUS_UNSUPPORTED; ++ } ++ ++ std::vector tokens; ++ const std::string prompt = rerank_prompt(session->stage_model->model, query, document); ++ if (!tokenize(llama_model_get_vocab(session->stage_model->model), prompt, tokens)) { ++ skippy_set_error(out_error, SKIPPY_STATUS_MODEL_ERROR, "failed to tokenize rerank input"); ++ return SKIPPY_STATUS_MODEL_ERROR; ++ } ++ status = validate_tokens(session, tokens.data(), tokens.size(), out_error); ++ if (status != SKIPPY_STATUS_OK) { ++ return status; ++ } ++ ++ const float * ranks = nullptr; ++ status = decode_embedding(session, tokens.data(), tokens.size(), &ranks, out_error); ++ if (status != SKIPPY_STATUS_OK) { ++ return status; ++ } ++ *out_score = ranks[0]; ++ *out_token_count = tokens.size(); ++ return skippy_success(out_error); ++} ++ ++enum skippy_status skippy_session_encode_prompt(skippy_session * session, ++ const llama_token * token_ids, ++ size_t token_count, ++ llama_token * out_decoder_start_token, ++ skippy_error ** out_error) { ++ enum skippy_status status = require_workload(session, SKIPPY_WORKLOAD_ENCODER_DECODER, out_error); ++ if (status != SKIPPY_STATUS_OK) { ++ return status; ++ } ++ if (out_decoder_start_token == nullptr) { ++ skippy_set_error(out_error, SKIPPY_STATUS_INVALID_ARGUMENT, "out_decoder_start_token is required"); ++ return SKIPPY_STATUS_INVALID_ARGUMENT; ++ } ++ status = validate_tokens(session, token_ids, token_count, out_error); ++ if (status != SKIPPY_STATUS_OK) { ++ return status; ++ } ++ if (token_count > static_cast(llama_n_ubatch(session->ctx))) { ++ skippy_set_error(out_error, SKIPPY_STATUS_INVALID_ARGUMENT, ++ "encoder input exceeds the physical batch capacity"); ++ return SKIPPY_STATUS_INVALID_ARGUMENT; ++ } ++ ++ clear_workload_state(session); ++ llama_batch batch = llama_batch_get_one(const_cast(token_ids), static_cast(token_count)); ++ if (llama_encode(session->ctx, batch) != 0) { ++ skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, "llama_encode failed for encoder-decoder workload"); ++ return SKIPPY_STATUS_RUNTIME_ERROR; ++ } ++ skippy_mark_context_work_pending(session->stage_model); ++ ++ llama_token start = llama_model_decoder_start_token(session->stage_model->model); ++ if (start == LLAMA_TOKEN_NULL) { ++ start = llama_vocab_bos(llama_model_get_vocab(session->stage_model->model)); ++ } ++ if (start == LLAMA_TOKEN_NULL) { ++ skippy_set_error(out_error, SKIPPY_STATUS_MODEL_ERROR, "model has no decoder start or BOS token"); ++ return SKIPPY_STATUS_MODEL_ERROR; ++ } ++ *out_decoder_start_token = start; ++ return skippy_success(out_error); ++} ++ ++} // extern "C" +diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt +index 5a45037bb..dc52da145 100644 +--- a/tests/CMakeLists.txt ++++ b/tests/CMakeLists.txt +@@ -150,6 +150,7 @@ endif () + + llama_build(test-recurrent-state-rollback.cpp) + llama_build(test-save-load-state.cpp) ++llama_build_and_test(test-skippy-rerank-template.cpp) + + if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) + llama_build_and_test(test-skippy-kv-page-export.cpp) +diff --git a/tests/test-skippy-rerank-template.cpp b/tests/test-skippy-rerank-template.cpp +new file mode 100644 +index 000000000..d6f9570ba +--- /dev/null ++++ b/tests/test-skippy-rerank-template.cpp +@@ -0,0 +1,34 @@ ++#include "../src/skippy/rerank_template.h" ++ ++#include ++#include ++ ++static bool expect_rendered( ++ std::string_view prompt_template, ++ std::string_view query, ++ std::string_view document, ++ std::string_view expected) { ++ const auto rendered = skippy_render_rerank_template(prompt_template, query, document); ++ if (std::string_view(rendered) == expected) { ++ return true; ++ } ++ std::fprintf(stderr, "rerank template rendered unexpected text: %s\n", rendered.c_str()); ++ return false; ++} ++ ++int main() { ++ if (!expect_rendered( ++ "Q={query}|D={document}|Q={query}", ++ "literal {document}", ++ "literal {query}", ++ "Q=literal {document}|D=literal {query}|Q=literal {document}")) { ++ return 1; ++ } ++ if (!expect_rendered("{query}{document}", "a", "b", "ab")) { ++ return 2; ++ } ++ if (!expect_rendered("no markers", "a", "b", "no markers")) { ++ return 3; ++ } ++ return 0; ++} +-- +2.54.0 (Apple Git-157) diff --git a/third_party/llama.cpp/patches/0044-skippy-configure-non-chat-model-loading.patch b/third_party/llama.cpp/patches/0044-skippy-configure-non-chat-model-loading.patch new file mode 100644 index 0000000000..049297a243 --- /dev/null +++ b/third_party/llama.cpp/patches/0044-skippy-configure-non-chat-model-loading.patch @@ -0,0 +1,54 @@ +From 8476626feba1706889564a7c3be71146e014cd5b Mon Sep 17 00:00:00 2001 +From: Mesh-LLM CI +Date: Sat, 12 Sep 2026 01:36:02 -0700 +Subject: [PATCH 35/38] skippy: configure non-chat model loading + +--- + src/skippy/model_loading.cpp | 23 +++++++++++++++++++++-- + 1 file changed, 21 insertions(+), 2 deletions(-) + +diff --git a/src/skippy/model_loading.cpp b/src/skippy/model_loading.cpp +index 874ca8842..76b102d67 100644 +--- a/src/skippy/model_loading.cpp ++++ b/src/skippy/model_loading.cpp +@@ -634,9 +634,19 @@ enum skippy_status skippy_finish_model_open( + } + } + +- const uint32_t lane_count = config != nullptr && config->lane_count > 0 ++ const bool encoder_decoder = llama_model_has_encoder(model) && llama_model_has_decoder(model); ++ const bool classifier_workload = model->cls != nullptr || model->cls_out != nullptr; ++ const bool embedding_workload = !encoder_decoder && ++ (classifier_workload || ++ (llama_model_has_encoder(model) || ++ (model->hparams.pooling_type != LLAMA_POOLING_TYPE_NONE && ++ model->hparams.pooling_type != LLAMA_POOLING_TYPE_UNSPECIFIED))); ++ const uint32_t configured_lane_count = config != nullptr && config->lane_count > 0 + ? static_cast(config->lane_count) + : 1; ++ // Encoder output is context-global in llama.cpp. Until that state can be ++ // isolated per sequence, serialize encoder-decoder work on one lane. ++ const uint32_t lane_count = encoder_decoder ? 1 : configured_lane_count; + uint32_t context_size_per_lane = 0; + uint32_t context_size_total = 0; + if (!skippy_context_capacity(config, lane_count, context_size_per_lane, context_size_total)) { +@@ -676,7 +686,16 @@ enum skippy_status skippy_finish_model_open( + params.type_k = config != nullptr && config->cache_type_k > 0 ? static_cast(config->cache_type_k) : GGML_TYPE_F16; + params.type_v = config != nullptr && config->cache_type_v > 0 ? static_cast(config->cache_type_v) : GGML_TYPE_F16; + params.flash_attn_type = config != nullptr ? static_cast(config->flash_attn_type) : LLAMA_FLASH_ATTN_TYPE_AUTO; +- params.embeddings = config != nullptr && config->filter_tensors_on_load && !config->include_output; ++ if (classifier_workload) { ++ params.pooling_type = LLAMA_POOLING_TYPE_RANK; ++ } ++ params.embeddings = embedding_workload || ++ (config != nullptr && config->filter_tensors_on_load && !config->include_output); ++ if (embedding_workload) { ++ // Non-causal embedding graphs must fit their logical batch in one ++ // physical batch, matching llama-embedding's runtime contract. ++ params.n_ubatch = params.n_batch; ++ } + const bool glm_dsa_op_timing_enabled = skippy_glm_dsa_op_timing_enabled(); + const bool glm_dsa_tensor_trace_enabled = skippy_glm_dsa_tensor_trace_enabled(); + if (model->arch == LLM_ARCH_GLM_DSA && (glm_dsa_op_timing_enabled || glm_dsa_tensor_trace_enabled)) { +-- +2.54.0 (Apple Git-157) diff --git a/third_party/llama.cpp/patches/0045-skippy-honor-vocab-suppress-tokens-in-sampling.patch b/third_party/llama.cpp/patches/0045-skippy-honor-vocab-suppress-tokens-in-sampling.patch new file mode 100644 index 0000000000..8b05c2a59b --- /dev/null +++ b/third_party/llama.cpp/patches/0045-skippy-honor-vocab-suppress-tokens-in-sampling.patch @@ -0,0 +1,347 @@ +From d151f2356d95101be739c343796009c2dc6a77b9 Mon Sep 17 00:00:00 2001 +From: Mesh-LLM CI +Date: Sat, 12 Sep 2026 04:07:46 -0700 +Subject: [PATCH 36/38] skippy: honor vocab suppress tokens in sampling + +--- + src/CMakeLists.txt | 1 + + src/skippy/sampling.cpp | 78 ++++++++++++------------- + src/skippy/sampling_internal.h | 13 +++++ + src/skippy/sampling_vocab.cpp | 63 ++++++++++++++++++++ + tests/CMakeLists.txt | 1 + + tests/test-skippy-sampling-suppress.cpp | 69 ++++++++++++++++++++++ + 6 files changed, 186 insertions(+), 39 deletions(-) + create mode 100644 src/skippy/sampling_vocab.cpp + create mode 100644 tests/test-skippy-sampling-suppress.cpp + +diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt +index 1d9049b4e..e0d41b550 100644 +--- a/src/CMakeLists.txt ++++ b/src/CMakeLists.txt +@@ -60,6 +60,7 @@ add_library(llama + skippy/model_package.cpp + skippy/package_tensor_binding.cpp + skippy/sampling.cpp ++ skippy/sampling_vocab.cpp + skippy/stage_slice_plan.cpp + skippy/stage_plan.cpp + skippy/state.cpp +diff --git a/src/skippy/sampling.cpp b/src/skippy/sampling.cpp +index 8c87bbe07..a900f28ce 100644 +--- a/src/skippy/sampling.cpp ++++ b/src/skippy/sampling.cpp +@@ -170,6 +170,8 @@ static llama_token skippy_greedy_sample_context_with_eog_policy( + } + const llama_vocab * vocab = llama_model_get_vocab(model); + const int32_t n_vocab = llama_vocab_n_tokens(vocab); ++ int32_t suppress_token_count = 0; ++ const llama_token * suppress_tokens = llama_vocab_get_suppress_tokens(vocab, &suppress_token_count); + llama_synchronize(ctx); + const float * logits = llama_get_logits_ith(ctx, index); + if (logits == nullptr) { +@@ -187,24 +189,17 @@ static llama_token skippy_greedy_sample_context_with_eog_policy( + &accelerated_best_logit, + &best_index, + static_cast(n_vocab)); +- if (!ignore_eog || !llama_vocab_is_eog(vocab, static_cast(best_index))) { ++ const bool accelerated_suppressed = suppress_tokens != nullptr && suppress_token_count > 0 && ++ std::find(suppress_tokens, suppress_tokens + suppress_token_count, ++ static_cast(best_index)) != suppress_tokens + suppress_token_count; ++ if (!accelerated_suppressed && ++ (!ignore_eog || !llama_vocab_is_eog(vocab, static_cast(best_index)))) { + return static_cast(best_index); + } + #endif + +- llama_token best = 0; +- float best_logit = -std::numeric_limits::infinity(); +- for (int32_t token = 0; token < n_vocab; ++token) { +- if (ignore_eog && llama_vocab_is_eog(vocab, token)) { +- continue; +- } +- if (logits[token] > best_logit) { +- best_logit = logits[token]; +- best = token; +- } +- } +- +- return best; ++ return skippy_greedy_sample_allowed_logits( ++ logits, vocab, n_vocab, suppress_tokens, suppress_token_count, ignore_eog); + } + + llama_token skippy_greedy_sample_context( +@@ -480,29 +475,35 @@ bool skippy_reset_reusable_sampling(skippy_session * session) { + static std::vector skippy_sampling_logit_biases( + skippy_session * session, + const skippy_sampling_config * sampling) { +- const uint32_t count = std::min(sampling->logit_bias_count, SKIPPY_MAX_LOGIT_BIAS); +- std::vector biases(sampling->logit_bias, sampling->logit_bias + count); +- const bool ignore_eos = sampling->ignore_eos != 0 || +- (sampling->flags & SKIPPY_SAMPLING_FLAG_IGNORE_EOS) != 0; +- if (!ignore_eos) { +- return biases; +- } ++ std::vector biases; ++ if (sampling != nullptr) { ++ const uint32_t count = std::min(sampling->logit_bias_count, SKIPPY_MAX_LOGIT_BIAS); ++ biases.assign(sampling->logit_bias, sampling->logit_bias + count); ++ } ++ const bool ignore_eos = sampling != nullptr && ++ (sampling->ignore_eos != 0 || ++ (sampling->flags & SKIPPY_SAMPLING_FLAG_IGNORE_EOS) != 0); + const llama_vocab * vocab = llama_model_get_vocab(session->stage_model->model); +- const int32_t token_count = llama_vocab_n_tokens(vocab); +- for (llama_token token = 0; token < token_count; ++token) { +- if (!llama_vocab_is_eog(vocab, token)) { +- continue; +- } +- const auto found = std::find_if(biases.begin(), biases.end(), [token](const llama_logit_bias & bias) { +- return bias.token == token; +- }); +- if (found == biases.end()) { +- biases.push_back({token, -INFINITY}); +- } else { +- found->bias = -INFINITY; ++ if (ignore_eos) { ++ const int32_t token_count = llama_vocab_n_tokens(vocab); ++ for (llama_token token = 0; token < token_count; ++token) { ++ if (!llama_vocab_is_eog(vocab, token)) { ++ continue; ++ } ++ const auto found = std::find_if(biases.begin(), biases.end(), [token](const llama_logit_bias & bias) { ++ return bias.token == token; ++ }); ++ if (found == biases.end()) { ++ biases.push_back({token, -INFINITY}); ++ } else { ++ found->bias = -INFINITY; ++ } + } + } +- return biases; ++ int32_t suppress_token_count = 0; ++ const llama_token * suppress_tokens = llama_vocab_get_suppress_tokens(vocab, &suppress_token_count); ++ return skippy_add_vocab_suppress_token_biases( ++ std::move(biases), suppress_tokens, suppress_token_count); + } + + static std::vector skippy_dry_sequence_breaker_storage( +@@ -607,11 +608,6 @@ llama_sampler * skippy_build_sampling_chain( + return nullptr; + } + +- if (skippy_sampling_is_greedy_equivalent(sampling)) { +- llama_sampler_chain_add(sampler, llama_sampler_init_greedy()); +- return sampler; +- } +- + const std::vector logit_biases = + skippy_sampling_logit_biases(session, sampling); + if (!logit_biases.empty()) { +@@ -623,6 +619,10 @@ llama_sampler * skippy_build_sampling_chain( + static_cast(logit_biases.size()), + logit_biases.data())); + } ++ if (skippy_sampling_is_greedy_equivalent(sampling)) { ++ llama_sampler_chain_add(sampler, llama_sampler_init_greedy()); ++ return sampler; ++ } + const uint32_t seed = sampling->seed == 0 ? LLAMA_DEFAULT_SEED : sampling->seed; + const float repeat_penalty = sampling->repeat_penalty == 0.0f + ? 1.0f +diff --git a/src/skippy/sampling_internal.h b/src/skippy/sampling_internal.h +index 9f819704f..4e60b256c 100644 +--- a/src/skippy/sampling_internal.h ++++ b/src/skippy/sampling_internal.h +@@ -8,10 +8,12 @@ + + #include + #include ++#include + + struct llama_context; + struct llama_model; + struct llama_sampler; ++struct llama_vocab; + struct skippy_session; + + void skippy_record_tokens(skippy_session * session, const llama_token * token_ids, size_t token_count); +@@ -29,8 +31,19 @@ enum skippy_status skippy_verify_token_batch( + size_t token_count, + skippy_error ** out_error); + llama_token skippy_greedy_sample_context(const llama_model * model, llama_context * ctx, int32_t index); ++llama_token skippy_greedy_sample_allowed_logits( ++ const float * logits, ++ const llama_vocab * vocab, ++ int32_t n_vocab, ++ const llama_token * suppress_tokens, ++ int32_t suppress_token_count, ++ bool ignore_eog); + llama_token skippy_greedy_sample_ith(skippy_session * session, int32_t index); + llama_sampler * skippy_build_sampling_chain(skippy_session * session, const skippy_sampling_config * sampling); ++std::vector skippy_add_vocab_suppress_token_biases( ++ std::vector biases, ++ const llama_token * suppress_tokens, ++ int32_t suppress_token_count); + void skippy_refresh_backend_sampling(skippy_session * session, const skippy_sampling_config * sampling); + bool skippy_reuse_plain_chat_sampling( + skippy_session * session, +diff --git a/src/skippy/sampling_vocab.cpp b/src/skippy/sampling_vocab.cpp +new file mode 100644 +index 000000000..1242ed105 +--- /dev/null ++++ b/src/skippy/sampling_vocab.cpp +@@ -0,0 +1,63 @@ ++#include "skippy/sampling_internal.h" ++ ++#include "llama.h" ++ ++#include ++#include ++#include ++#include ++#include ++ ++std::vector skippy_add_vocab_suppress_token_biases( ++ std::vector biases, ++ const llama_token * suppress_tokens, ++ int32_t suppress_token_count) { ++ if (suppress_tokens == nullptr || suppress_token_count <= 0) { ++ return biases; ++ } ++ const llama_token * suppress_end = suppress_tokens + suppress_token_count; ++ biases.erase(std::remove_if(biases.begin(), biases.end(), [=](const llama_logit_bias & bias) { ++ return std::find(suppress_tokens, suppress_end, bias.token) != suppress_end; ++ }), biases.end()); ++ biases.reserve(biases.size() + suppress_token_count); ++ for (const llama_token * token = suppress_tokens; token != suppress_end; ++token) { ++ biases.push_back({*token, -INFINITY}); ++ } ++ return biases; ++} ++ ++llama_token skippy_greedy_sample_allowed_logits( ++ const float * logits, ++ const llama_vocab * vocab, ++ int32_t n_vocab, ++ const llama_token * suppress_tokens, ++ int32_t suppress_token_count, ++ bool ignore_eog) { ++ if (logits == nullptr || n_vocab <= 0) { ++ return LLAMA_TOKEN_NULL; ++ } ++ std::vector suppressed; ++ if (suppress_tokens != nullptr && suppress_token_count > 0) { ++ suppressed.resize(static_cast(n_vocab), 0); ++ for (int32_t i = 0; i < suppress_token_count; ++i) { ++ const llama_token token = suppress_tokens[i]; ++ if (token >= 0 && token < n_vocab) { ++ suppressed[static_cast(token)] = 1; ++ } ++ } ++ } ++ ++ llama_token best = LLAMA_TOKEN_NULL; ++ float best_logit = -std::numeric_limits::infinity(); ++ for (llama_token token = 0; token < n_vocab; ++token) { ++ if ((!suppressed.empty() && suppressed[static_cast(token)] != 0) || ++ (ignore_eog && vocab != nullptr && llama_vocab_is_eog(vocab, token))) { ++ continue; ++ } ++ if (best == LLAMA_TOKEN_NULL || logits[token] > best_logit) { ++ best_logit = logits[token]; ++ best = token; ++ } ++ } ++ return best; ++} +diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt +index dc52da145..2b2aae216 100644 +--- a/tests/CMakeLists.txt ++++ b/tests/CMakeLists.txt +@@ -153,6 +153,7 @@ llama_build(test-save-load-state.cpp) + llama_build_and_test(test-skippy-rerank-template.cpp) + + if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) ++ llama_build_and_test(test-skippy-sampling-suppress.cpp) + llama_build_and_test(test-skippy-kv-page-export.cpp) + llama_build_and_test(test-skippy-verify-checkpoint-retirement.cpp) + llama_build_and_test(test-skippy-kv-cells-contiguous.cpp) +diff --git a/tests/test-skippy-sampling-suppress.cpp b/tests/test-skippy-sampling-suppress.cpp +new file mode 100644 +index 000000000..4cf702bef +--- /dev/null ++++ b/tests/test-skippy-sampling-suppress.cpp +@@ -0,0 +1,69 @@ ++#include "../src/skippy/sampling_internal.h" ++ ++#include "llama.h" ++ ++#include ++#include ++#include ++ ++static bool suppressed(const llama_logit_bias & bias) { ++ return std::isinf(bias.bias) && std::signbit(bias.bias); ++} ++ ++int main() { ++ const llama_token suppress_tokens[] = {2, 3}; ++ const std::vector caller_biases = { ++ {2, 100.0f}, ++ {4, -1.0f}, ++ {2, 50.0f}, ++ }; ++ auto merged = skippy_add_vocab_suppress_token_biases( ++ caller_biases, suppress_tokens, 2); ++ if (merged.size() != 3 || merged[0].token != 4 || merged[0].bias != -1.0f || ++ merged[1].token != 2 || !suppressed(merged[1]) || ++ merged[2].token != 3 || !suppressed(merged[2])) { ++ std::fprintf(stderr, "vocab suppress tokens did not override caller logit biases\n"); ++ return 1; ++ } ++ ++ llama_token_data candidates[] = { ++ {0, 1.0f, 0.0f}, ++ {1, 2.0f, 0.0f}, ++ {2, 100.0f, 0.0f}, ++ {3, 50.0f, 0.0f}, ++ {4, 4.0f, 0.0f}, ++ }; ++ llama_token_data_array view = {candidates, 5, -1, false}; ++ llama_sampler * chain = llama_sampler_chain_init(llama_sampler_chain_default_params()); ++ llama_sampler_chain_add(chain, llama_sampler_init_logit_bias(5, merged.size(), merged.data())); ++ llama_sampler_chain_add(chain, llama_sampler_init_greedy()); ++ llama_sampler_apply(chain, &view); ++ llama_sampler_free(chain); ++ if (!std::isinf(candidates[2].logit) || !std::signbit(candidates[2].logit) || ++ !std::isinf(candidates[3].logit) || !std::signbit(candidates[3].logit) || ++ candidates[4].logit != 3.0f || view.selected != 4) { ++ std::fprintf(stderr, "vocab suppression did not reach the sampling chain\n"); ++ return 2; ++ } ++ ++ const float logits[] = {1.0f, 2.0f, 100.0f, 50.0f, 4.0f}; ++ if (skippy_greedy_sample_allowed_logits(logits, nullptr, 5, suppress_tokens, 2, false) != 4 || ++ skippy_greedy_sample_allowed_logits(logits, nullptr, 5, nullptr, 0, false) != 2) { ++ std::fprintf(stderr, "greedy fast path ignored vocab suppression\n"); ++ return 3; ++ } ++ const llama_token all_tokens[] = {0, 1, 2, 3, 4}; ++ const llama_token invalid_token[] = {99}; ++ if (skippy_greedy_sample_allowed_logits(logits, nullptr, 5, all_tokens, 5, false) != LLAMA_TOKEN_NULL || ++ skippy_greedy_sample_allowed_logits(logits, nullptr, 5, invalid_token, 1, false) != 2) { ++ std::fprintf(stderr, "greedy fast path mishandled exhausted or invalid suppression\n"); ++ return 4; ++ } ++ ++ merged = skippy_add_vocab_suppress_token_biases(caller_biases, nullptr, 0); ++ if (merged.size() != caller_biases.size()) { ++ std::fprintf(stderr, "models without suppress tokens lost caller logit biases\n"); ++ return 5; ++ } ++ return 0; ++} +-- +2.54.0 (Apple Git-157) diff --git a/third_party/llama.cpp/patches/0046-fix-skippy-reject-exhausted-vocab-sampling.patch b/third_party/llama.cpp/patches/0046-fix-skippy-reject-exhausted-vocab-sampling.patch new file mode 100644 index 0000000000..d17e21d005 --- /dev/null +++ b/third_party/llama.cpp/patches/0046-fix-skippy-reject-exhausted-vocab-sampling.patch @@ -0,0 +1,89 @@ +From 041b032747cca40413780cbf4f8015e2a70ce1d3 Mon Sep 17 00:00:00 2001 +From: scama + +Date: Sun, 13 Sep 2026 10:45:49 +1000 +Subject: [PATCH 37/38] fix(skippy): reject exhausted vocab sampling + +--- + src/skippy/execution-single.cpp | 6 ++++++ + src/skippy/session.cpp | 5 +++++ + src/skippy/speculative_decoding.cpp | 6 ++++++ + src/skippy/verification.cpp | 6 ++++++ + 4 files changed, 23 insertions(+) + +diff --git a/src/skippy/execution-single.cpp b/src/skippy/execution-single.cpp +index c0153bf77..357222c13 100644 +--- a/src/skippy/execution-single.cpp ++++ b/src/skippy/execution-single.cpp +@@ -282,6 +282,12 @@ enum skippy_status skippy_verify_tokens( + const int32_t n_tokens = static_cast(token_count); + for (int32_t i = 0; i < n_tokens; ++i) { + output_tokens[i] = skippy_greedy_sample_ith(session, i); ++ if (output_tokens[i] == LLAMA_TOKEN_NULL) { ++ *out_token_count = 0; ++ skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, ++ "verification produced no eligible token after vocabulary suppression"); ++ return SKIPPY_STATUS_RUNTIME_ERROR; ++ } + } + } + return status; +diff --git a/src/skippy/session.cpp b/src/skippy/session.cpp +index e3b8b5716..8f2c44df2 100644 +--- a/src/skippy/session.cpp ++++ b/src/skippy/session.cpp +@@ -266,6 +266,11 @@ enum skippy_status skippy_session_sample_current( + return SKIPPY_STATUS_INVALID_ARGUMENT; + } + *out_predicted_token = skippy_sample_token(session, sampling); ++ if (*out_predicted_token == LLAMA_TOKEN_NULL) { ++ skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, ++ "sampling produced no eligible token after vocabulary suppression"); ++ return SKIPPY_STATUS_RUNTIME_ERROR; ++ } + return skippy_success(out_error); + } + +diff --git a/src/skippy/speculative_decoding.cpp b/src/skippy/speculative_decoding.cpp +index 88b3ba0a0..b28ef9096 100644 +--- a/src/skippy/speculative_decoding.cpp ++++ b/src/skippy/speculative_decoding.cpp +@@ -460,6 +460,9 @@ enum skippy_status skippy_mtp_propose_next( + } + + token = skippy_greedy_sample_context(llama_get_model(mtp_ctx), mtp_ctx, -1); ++ if (token == LLAMA_TOKEN_NULL) { ++ break; ++ } + if (out_mtp_draft != nullptr) { + out_mtp_draft->token_ids[token_count] = token; + } +@@ -520,6 +523,9 @@ enum skippy_status skippy_mtp_propose_next( + } + + token = skippy_greedy_sample_context(llama_get_model(mtp_ctx), mtp_ctx, -1); ++ if (token == LLAMA_TOKEN_NULL) { ++ break; ++ } + if (out_mtp_draft != nullptr) { + out_mtp_draft->token_ids[token_count] = token; + } +diff --git a/src/skippy/verification.cpp b/src/skippy/verification.cpp +index b91d87021..c14e7a0a3 100644 +--- a/src/skippy/verification.cpp ++++ b/src/skippy/verification.cpp +@@ -230,6 +230,12 @@ enum skippy_status skippy_verify_tokens_frame_sampled( + skippy_record_tokens(session, &token_ids[i], 1); + const int32_t logits_index = static_cast(i); + output_tokens[i] = skippy_sample_token_ith(session, sampling, logits_index); ++ if (output_tokens[i] == LLAMA_TOKEN_NULL) { ++ *out_token_count = 0; ++ skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, ++ "sampled verification produced no eligible token after vocabulary suppression"); ++ return SKIPPY_STATUS_RUNTIME_ERROR; ++ } + if (stop_after_first_mismatch && i + 1 < token_count && output_tokens[i] != token_ids[i + 1]) { + // Later rows are conditioned on a rejected token and are not + // authoritative predictions. Stop advancing the sampler and +-- +2.54.0 (Apple Git-157) diff --git a/third_party/llama.cpp/patches/0047-skippy-validate-sampling-across-all-execution-bounda.patch b/third_party/llama.cpp/patches/0047-skippy-validate-sampling-across-all-execution-bounda.patch new file mode 100644 index 0000000000..1d7416b5ad --- /dev/null +++ b/third_party/llama.cpp/patches/0047-skippy-validate-sampling-across-all-execution-bounda.patch @@ -0,0 +1,306 @@ +From 22cb4b643a44a7ee291245d81306a26b6977fae9 Mon Sep 17 00:00:00 2001 +From: Mesh-LLM CI +Date: Sat, 12 Sep 2026 18:24:13 -0700 +Subject: [PATCH 38/38] skippy: validate sampling across all execution + boundaries + +--- + src/skippy/execution-batch.cpp | 15 ++++++++--- + src/skippy/execution-single.cpp | 36 +++++++++++++++++-------- + src/skippy/sampling_internal.h | 3 +++ + src/skippy/sampling_vocab.cpp | 11 ++++++++ + src/skippy/session.cpp | 9 ++----- + src/skippy/speculative_decoding.cpp | 27 ++++++++++++++----- + src/skippy/verification.cpp | 9 +++---- + tests/test-skippy-sampling-suppress.cpp | 21 +++++++++++++++ + 8 files changed, 98 insertions(+), 33 deletions(-) + +diff --git a/src/skippy/execution-batch.cpp b/src/skippy/execution-batch.cpp +index b66e25499..bf7cc1ecb 100644 +--- a/src/skippy/execution-batch.cpp ++++ b/src/skippy/execution-batch.cpp +@@ -340,11 +340,14 @@ enum skippy_status skippy_iteration_batch_sampled( + // the sparse-row translation internally. + const int32_t logits_index = + static_cast(request_offsets[request_index + 1] - 1); +- out_sampled_request_indexes[sampled_output_index] = request_index; +- out_predicted_tokens[sampled_output_index] = skippy_sample_token_ith( ++ const skippy_status sample_status = skippy_store_sampled_token(skippy_sample_token_ith( + request.session, + request.sampling, +- logits_index); ++ logits_index), &out_predicted_tokens[sampled_output_index], out_error); ++ if (sample_status != SKIPPY_STATUS_OK) { ++ return sample_status; ++ } ++ out_sampled_request_indexes[sampled_output_index] = request_index; + sampled_output_index += 1; + } + +@@ -680,7 +683,11 @@ enum skippy_status skippy_decode_step_frame_batch_sampled( + } + if (request_logits) { + const skippy_sampling_config * request_sampling = sampling != nullptr ? sampling[i] : nullptr; +- out_predicted_tokens[i] = skippy_sample_token_ith(session, request_sampling, i); ++ const skippy_status sample_status = skippy_store_sampled_token( ++ skippy_sample_token_ith(session, request_sampling, i), &out_predicted_tokens[i], out_error); ++ if (sample_status != SKIPPY_STATUS_OK) { ++ return sample_status; ++ } + } else { + out_predicted_tokens[i] = -1; + } +diff --git a/src/skippy/execution-single.cpp b/src/skippy/execution-single.cpp +index 357222c13..967a6b1ed 100644 +--- a/src/skippy/execution-single.cpp ++++ b/src/skippy/execution-single.cpp +@@ -119,7 +119,8 @@ enum skippy_status skippy_decode_step_sampled( + } + + if (out_predicted_token != nullptr) { +- *out_predicted_token = skippy_sample_token(session, sampling); ++ return skippy_store_sampled_token( ++ skippy_sample_token(session, sampling), out_predicted_token, out_error); + } + + return skippy_success(out_error); +@@ -233,7 +234,11 @@ enum skippy_status skippy_decode_batch_sampled( + session->n_past += 1; + skippy_record_tokens(session, &token_ids[i], 1); + const skippy_sampling_config * request_sampling = sampling != nullptr ? sampling[i] : nullptr; +- out_predicted_tokens[i] = skippy_sample_token_ith(session, request_sampling, i); ++ const skippy_status sample_status = skippy_store_sampled_token( ++ skippy_sample_token_ith(session, request_sampling, i), &out_predicted_tokens[i], out_error); ++ if (sample_status != SKIPPY_STATUS_OK) { ++ return sample_status; ++ } + } + + return skippy_success(out_error); +@@ -281,12 +286,11 @@ enum skippy_status skippy_verify_tokens( + if (status == SKIPPY_STATUS_OK) { + const int32_t n_tokens = static_cast(token_count); + for (int32_t i = 0; i < n_tokens; ++i) { +- output_tokens[i] = skippy_greedy_sample_ith(session, i); +- if (output_tokens[i] == LLAMA_TOKEN_NULL) { ++ status = skippy_store_sampled_token( ++ skippy_greedy_sample_ith(session, i), &output_tokens[i], out_error); ++ if (status != SKIPPY_STATUS_OK) { + *out_token_count = 0; +- skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, +- "verification produced no eligible token after vocabulary suppression"); +- return SKIPPY_STATUS_RUNTIME_ERROR; ++ return status; + } + } + } +@@ -370,9 +374,15 @@ static enum skippy_status skippy_prefill_chunk_frame_impl( + return status; + } + +- if (out_predicted_token != nullptr) { +- *out_predicted_token = session->stage_model->config.include_output ? +- skippy_sample_token(session, sampling) : -1; ++ if (out_predicted_token != nullptr) { ++ *out_predicted_token = -1; ++ if (session->stage_model->config.include_output) { ++ status = skippy_store_sampled_token( ++ skippy_sample_token(session, sampling), out_predicted_token, out_error); ++ if (status != SKIPPY_STATUS_OK) { ++ return status; ++ } ++ } + } + + return skippy_copy_output_activation_frame( +@@ -564,7 +571,14 @@ enum skippy_status skippy_decode_step_frame_sampled( + } + + if (out_predicted_token != nullptr) { +- *out_predicted_token = session->stage_model->config.include_output ? skippy_sample_token(session, sampling) : -1; ++ *out_predicted_token = -1; ++ if (session->stage_model->config.include_output) { ++ status = skippy_store_sampled_token( ++ skippy_sample_token(session, sampling), out_predicted_token, out_error); ++ if (status != SKIPPY_STATUS_OK) { ++ return status; ++ } ++ } + } + + return skippy_copy_output_activation_frame( +diff --git a/src/skippy/sampling_internal.h b/src/skippy/sampling_internal.h +index 4e60b256c..0c4e54724 100644 +--- a/src/skippy/sampling_internal.h ++++ b/src/skippy/sampling_internal.h +@@ -39,6 +39,9 @@ llama_token skippy_greedy_sample_allowed_logits( + int32_t suppress_token_count, + bool ignore_eog); + llama_token skippy_greedy_sample_ith(skippy_session * session, int32_t index); ++// Keep exhausted sampling out of public outputs and subsequent decode inputs. ++enum skippy_status skippy_store_sampled_token( ++ llama_token token, llama_token * output, skippy_error ** out_error); + llama_sampler * skippy_build_sampling_chain(skippy_session * session, const skippy_sampling_config * sampling); + std::vector skippy_add_vocab_suppress_token_biases( + std::vector biases, +diff --git a/src/skippy/sampling_vocab.cpp b/src/skippy/sampling_vocab.cpp +index 1242ed105..baf83db44 100644 +--- a/src/skippy/sampling_vocab.cpp ++++ b/src/skippy/sampling_vocab.cpp +@@ -1,4 +1,5 @@ + #include "skippy/sampling_internal.h" ++#include "skippy/errors.h" + + #include "llama.h" + +@@ -8,6 +9,16 @@ + #include + #include + ++enum skippy_status skippy_store_sampled_token( ++ llama_token token, llama_token * output, skippy_error ** out_error) { ++ if (token < 0) { ++ skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, "sampling produced no allowed token"); ++ return SKIPPY_STATUS_RUNTIME_ERROR; ++ } ++ *output = token; ++ return skippy_success(out_error); ++} ++ + std::vector skippy_add_vocab_suppress_token_biases( + std::vector biases, + const llama_token * suppress_tokens, +diff --git a/src/skippy/session.cpp b/src/skippy/session.cpp +index 8f2c44df2..386847222 100644 +--- a/src/skippy/session.cpp ++++ b/src/skippy/session.cpp +@@ -265,13 +265,8 @@ enum skippy_status skippy_session_sample_current( + skippy_set_error(out_error, SKIPPY_STATUS_INVALID_ARGUMENT, "session and out_predicted_token are required"); + return SKIPPY_STATUS_INVALID_ARGUMENT; + } +- *out_predicted_token = skippy_sample_token(session, sampling); +- if (*out_predicted_token == LLAMA_TOKEN_NULL) { +- skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, +- "sampling produced no eligible token after vocabulary suppression"); +- return SKIPPY_STATUS_RUNTIME_ERROR; +- } +- return skippy_success(out_error); ++ return skippy_store_sampled_token( ++ skippy_sample_token(session, sampling), out_predicted_token, out_error); + } + + enum skippy_status skippy_session_configure_chat_sampling( +diff --git a/src/skippy/speculative_decoding.cpp b/src/skippy/speculative_decoding.cpp +index b28ef9096..321852e87 100644 +--- a/src/skippy/speculative_decoding.cpp ++++ b/src/skippy/speculative_decoding.cpp +@@ -295,8 +295,16 @@ enum skippy_status skippy_mtp_sync_target_inputs( + for (int32_t out_i = 0; out_i + 1 < n_decode; ++out_i) { + batch.token[out_i] = batch.token[out_i + 1]; + } +- batch.token[n_decode - 1] = skippy_greedy_sample_context( +- llama_get_model(mtp_ctx), mtp_ctx, -1); ++ const skippy_status sample_status = skippy_store_sampled_token( ++ skippy_greedy_sample_context(llama_get_model(mtp_ctx), mtp_ctx, -1), ++ &batch.token[n_decode - 1], out_error); ++ if (sample_status != SKIPPY_STATUS_OK) { ++ skippy_mtp_clear_session_state(session); ++ std::free(batch.token); ++ batch.token = nullptr; ++ llama_batch_free(batch); ++ return sample_status; ++ } + } + if (batch.token != nullptr) { + std::free(batch.token); +@@ -460,8 +468,13 @@ enum skippy_status skippy_mtp_propose_next( + } + + token = skippy_greedy_sample_context(llama_get_model(mtp_ctx), mtp_ctx, -1); +- if (token == LLAMA_TOKEN_NULL) { +- break; ++ const skippy_status sample_status = skippy_store_sampled_token(token, &token, out_error); ++ if (sample_status != SKIPPY_STATUS_OK) { ++ skippy_mtp_clear_session_state(session); ++ std::free(batch.token); ++ batch.token = nullptr; ++ llama_batch_free(batch); ++ return sample_status; + } + if (out_mtp_draft != nullptr) { + out_mtp_draft->token_ids[token_count] = token; +@@ -523,8 +536,10 @@ enum skippy_status skippy_mtp_propose_next( + } + + token = skippy_greedy_sample_context(llama_get_model(mtp_ctx), mtp_ctx, -1); +- if (token == LLAMA_TOKEN_NULL) { +- break; ++ const skippy_status sample_status = skippy_store_sampled_token(token, &token, out_error); ++ if (sample_status != SKIPPY_STATUS_OK) { ++ skippy_mtp_clear_session_state(session); ++ return sample_status; + } + if (out_mtp_draft != nullptr) { + out_mtp_draft->token_ids[token_count] = token; +diff --git a/src/skippy/verification.cpp b/src/skippy/verification.cpp +index c14e7a0a3..09a8a2012 100644 +--- a/src/skippy/verification.cpp ++++ b/src/skippy/verification.cpp +@@ -229,12 +229,11 @@ enum skippy_status skippy_verify_tokens_frame_sampled( + for (size_t i = 0; i < token_count; ++i) { + skippy_record_tokens(session, &token_ids[i], 1); + const int32_t logits_index = static_cast(i); +- output_tokens[i] = skippy_sample_token_ith(session, sampling, logits_index); +- if (output_tokens[i] == LLAMA_TOKEN_NULL) { ++ status = skippy_store_sampled_token( ++ skippy_sample_token_ith(session, sampling, logits_index), &output_tokens[i], out_error); ++ if (status != SKIPPY_STATUS_OK) { + *out_token_count = 0; +- skippy_set_error(out_error, SKIPPY_STATUS_RUNTIME_ERROR, +- "sampled verification produced no eligible token after vocabulary suppression"); +- return SKIPPY_STATUS_RUNTIME_ERROR; ++ return status; + } + if (stop_after_first_mismatch && i + 1 < token_count && output_tokens[i] != token_ids[i + 1]) { + // Later rows are conditioned on a rejected token and are not +diff --git a/tests/test-skippy-sampling-suppress.cpp b/tests/test-skippy-sampling-suppress.cpp +index 4cf702bef..3f797cbe8 100644 +--- a/tests/test-skippy-sampling-suppress.cpp ++++ b/tests/test-skippy-sampling-suppress.cpp +@@ -4,6 +4,7 @@ + + #include + #include ++#include + #include + + static bool suppressed(const llama_logit_bias & bias) { +@@ -60,6 +61,26 @@ int main() { + return 4; + } + ++ llama_token output = 42; ++ skippy_error * error = nullptr; ++ const llama_token exhausted = skippy_greedy_sample_allowed_logits( ++ logits, nullptr, 5, all_tokens, 5, false); ++ if (skippy_store_sampled_token(exhausted, &output, &error) != SKIPPY_STATUS_RUNTIME_ERROR || ++ output != 42 || error == nullptr || ++ std::strstr(error->message, "no allowed token") == nullptr) { ++ std::fprintf(stderr, "exhausted sampling leaked into the output buffer\n"); ++ skippy_error_free(error); ++ return 6; ++ } ++ skippy_error_free(error); ++ error = nullptr; ++ if (skippy_store_sampled_token(0, &output, &error) != SKIPPY_STATUS_OK || ++ output != 0 || error != nullptr) { ++ std::fprintf(stderr, "valid token zero was rejected\n"); ++ skippy_error_free(error); ++ return 7; ++ } ++ + merged = skippy_add_vocab_suppress_token_biases(caller_biases, nullptr, 0); + if (merged.size() != caller_biases.size()) { + std::fprintf(stderr, "models without suppress tokens lost caller logit biases\n"); +-- +2.54.0 (Apple Git-157) diff --git a/tools/xtask/data/console_print_allowlist.json b/tools/xtask/data/console_print_allowlist.json index cf79c6119d..d2fbcab11b 100644 --- a/tools/xtask/data/console_print_allowlist.json +++ b/tools/xtask/data/console_print_allowlist.json @@ -4453,13 +4453,13 @@ "macro_name": "eprintln!" }, { - "line": 348, + "line": 307, "macro_name": "eprintln!" } ], "crates/skippy-server/src/http.rs": [ { - "line": 216, + "line": 223, "macro_name": "println!" } ], @@ -4471,27 +4471,27 @@ ], "crates/skippy-server/src/runtime_state.rs": [ { - "line": 819, + "line": 890, "macro_name": "eprintln!" }, { - "line": 958, + "line": 1029, "macro_name": "eprintln!" }, { - "line": 1001, + "line": 1072, "macro_name": "eprintln!" }, { - "line": 1039, + "line": 1110, "macro_name": "eprintln!" }, { - "line": 1043, + "line": 1114, "macro_name": "eprintln!" }, { - "line": 1048, + "line": 1119, "macro_name": "eprintln!" } ] diff --git a/website/src/docs/pages/skippy-api.md b/website/src/docs/pages/skippy-api.md index 1931fae959..5fd2dfdba5 100644 --- a/website/src/docs/pages/skippy-api.md +++ b/website/src/docs/pages/skippy-api.md @@ -9,7 +9,7 @@ description: Generated reference for the capability-oriented Skippy C ABI. This reference is generated from the patched llama.cpp public headers. It documents the native C ABI used by Skippy's Rust FFI layer and staged runtime. The ABI is experimental and versioned for lockstep native/Rust builds. -Current generated surface: **15 headers** and **99 exported functions**. +Current generated surface: **16 headers** and **103 exported functions**. ## Quick navigation @@ -185,6 +185,15 @@ Current generated surface: **15 headers** and **99 exported functions**. skippy_parse_chat_response_json +
+ workloads.h4 functions +
+ skippy_model_workload_info_v1 + skippy_session_embed + skippy_session_rerank + skippy_session_encode_prompt +
+
@@ -216,6 +225,7 @@ Capability consumers can include a narrower header: | `include/skippy/stage_plan.h` | Metadata-only construction and inspection of guarded stage plans. | | `include/skippy/state.h` | Moves KV, recurrent, checkpoint, and resident-prefix state. | | `include/skippy/tokenization.h` | Token, detokenization, chat-template, and chat-response helpers. | +| `include/skippy/workloads.h` | Full-model embedding, reranking, and encoder-decoder execution. These operations deliberately reject filtered stage models. Their explicit workload descriptor lets callers fail closed instead of inferring support from a model name or architecture family. | ## ABI conventions @@ -1668,13 +1678,75 @@ SKIPPY_COMMON_API enum skippy_status skippy_parse_chat_response_json( ↩ Back to function index + +### `workloads.h` + + +#### `skippy_model_workload_info_v1` + +Describes the workload implemented by an opened model. + +```cpp +LLAMA_API enum skippy_status skippy_model_workload_info_v1( + const struct skippy_model * model, + struct skippy_workload_info_v1 * out_info, + struct skippy_error ** out_error); +``` + + +#### `skippy_session_embed` + +Computes one normalized embedding for a tokenized input. + +```cpp +LLAMA_API enum skippy_status skippy_session_embed( + struct skippy_session * session, + const llama_token * token_ids, + size_t token_count, + float * output, + size_t output_capacity, + size_t * out_dimensions, + struct skippy_error ** out_error); +``` + + +#### `skippy_session_rerank` + +Computes one scalar relevance score for a query-document pair. + +```cpp +LLAMA_API enum skippy_status skippy_session_rerank( + struct skippy_session * session, + const char * query, + const char * document, + float * out_score, + size_t * out_token_count, + struct skippy_error ** out_error); +``` + + +#### `skippy_session_encode_prompt` + +Encodes an encoder-decoder prompt and returns its first decoder token. + +```cpp +LLAMA_API enum skippy_status skippy_session_encode_prompt( + struct skippy_session * session, + const llama_token * token_ids, + size_t token_count, + llama_token * out_decoder_start_token, + struct skippy_error ** out_error); +``` + +↩ Back to function index + ## Native declarations The headers also define the following enums, structs, opaque handles, and ABI constants: - `activation.h`: `skippy_activation_dtype`, `skippy_activation_layout`, `skippy_activation_boundary_desc`, `skippy_activation_desc`, `SKIPPY_ACTIVATION_BOUNDARY_DESC_VERSION = 1`, `SKIPPY_ACTIVATION_SIDEBAND_TOKEN_IDS = (UINT64_C(1) << 0)`, `SKIPPY_ACTIVATION_FLAG_RWKV7_V_FIRST = (UINT64_C(1) << 0)`, `SKIPPY_ACTIVATION_FLAG_GEMMA3N_ALTUP = (UINT64_C(1) << 1)`, `SKIPPY_ACTIVATION_FLAG_INKLING_MTP_EMBD = (UINT64_C(1) << 2)`, `SKIPPY_ACTIVATION_FLAG_GLM_DSA_TOP_K = (UINT64_C(1) << 3)`, `SKIPPY_ACTIVATION_FLAG_KIMI_K3_RESIDUAL = (UINT64_C(1) << 4)` -- `common.h`: `skippy_feature`, `skippy_status`, `skippy_error`, `skippy_abi_version`, `SKIPPY_ABI_VERSION_MAJOR = 0`, `SKIPPY_ABI_VERSION_MINOR = 1`, `SKIPPY_ABI_VERSION_PATCH = 54`, `SKIPPY_FEATURE_RUNTIME_EVENT_REPORTER = ((uint64_t)1 << 31)`, `SKIPPY_FEATURE_MODEL_LOAD_EVENTS_V2 = ((uint64_t)1 << 32)`, `SKIPPY_FEATURE_KV_EVENTS = ((uint64_t)1 << 33)`, `SKIPPY_FEATURE_DEVICE_EVENTS = ((uint64_t)1 << 34)`, `SKIPPY_FEATURE_DIAGNOSTIC_EVENTS = ((uint64_t)1 << 35)`, `SKIPPY_FEATURE_UNLOAD_EVENTS = ((uint64_t)1 << 36)` +- `common.h`: `skippy_feature`, `skippy_status`, `skippy_error`, `skippy_abi_version`, `SKIPPY_ABI_VERSION_MAJOR = 0`, `SKIPPY_ABI_VERSION_MINOR = 1`, `SKIPPY_ABI_VERSION_PATCH = 55`, `SKIPPY_FEATURE_RUNTIME_EVENT_REPORTER = ((uint64_t)1 << 31)`, `SKIPPY_FEATURE_MODEL_LOAD_EVENTS_V2 = ((uint64_t)1 << 32)`, `SKIPPY_FEATURE_KV_EVENTS = ((uint64_t)1 << 33)`, `SKIPPY_FEATURE_DEVICE_EVENTS = ((uint64_t)1 << 34)`, `SKIPPY_FEATURE_DIAGNOSTIC_EVENTS = ((uint64_t)1 << 35)`, `SKIPPY_FEATURE_UNLOAD_EVENTS = ((uint64_t)1 << 36)`, `SKIPPY_FEATURE_NON_CHAT_WORKLOADS = ((uint64_t)1 << 37)` - `devices.h`: `skippy_backend_device_type`, `skippy_backend_device_cap`, `skippy_backend_device` - `events.h`: `skippy_runtime_event_v1`, `skippy_runtime_event_reporter_v1`, `SKIPPY_RUNTIME_EVENT_V1_ABI_VERSION = 1` - `execution.h`: `skippy_iteration_request` @@ -1686,5 +1758,6 @@ The headers also define the following enums, structs, opaque handles, and ABI co - `speculative_decoding.h`: `skippy_ngram_cache`, `skippy_native_mtp_draft`, `SKIPPY_NATIVE_MTP_MAX_DRAFT_TOKENS = 8` - `stage_plan.h`: `skippy_stage_planner`, `skippy_stage_plan`, `skippy_stage_plan_string_ref_v1`, `skippy_stage_planner_tensor_v1`, `skippy_stage_planner_profile_v1`, `skippy_stage_planner_config_v1`, `skippy_stage_plan_value_kind`, `skippy_stage_plan_state_kind`, `skippy_stage_plan_state_access`, `skippy_stage_plan_desc_v1`, `skippy_stage_plan_profile_desc_v1`, `skippy_stage_plan_value_desc_v1`, `skippy_stage_plan_state_desc_v1`, `SKIPPY_STAGE_PLANNER_CONFIG_V1_ABI_VERSION = 1`, `SKIPPY_STAGE_PLANNER_TENSOR_V1_ABI_VERSION = 1`, `SKIPPY_STAGE_PLANNER_PROFILE_V1_ABI_VERSION = 1`, `SKIPPY_STAGE_PLAN_DESC_V1_ABI_VERSION = 1`, `SKIPPY_STAGE_PLAN_PROFILE_DESC_V1_ABI_VERSION = 1`, `SKIPPY_STAGE_PLAN_VALUE_DESC_V1_ABI_VERSION = 1`, `SKIPPY_STAGE_PLAN_STATE_DESC_V1_ABI_VERSION = 1`, `SKIPPY_STAGE_PLAN_MAX_DIMS = 4` - `state.h`: `skippy_kv_page_flag`, `skippy_kv_page_codec`, `skippy_kv_page_component_role`, `skippy_kv_page_component_desc`, `skippy_kv_page_desc` +- `workloads.h`: `skippy_model`, `skippy_session`, `skippy_workload_kind`, `skippy_workload_pooling`, `skippy_workload_info_v1`, `SKIPPY_WORKLOAD_INFO_V1_ABI_VERSION = 1` Source directory: `include/skippy/`. Regenerate this page after changing any public header or exported function.