diff --git a/Cargo.lock b/Cargo.lock index 6d72366..26b0bc7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -426,7 +426,7 @@ dependencies = [ "candle-kernels", "candle-metal-kernels", "candle-ug", - "cudarc 0.19.9", + "cudarc 0.19.10", "float8", "gemm 0.19.0", "half", @@ -441,7 +441,7 @@ dependencies = [ "rand_distr", "rayon", "safetensors 0.8.0", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokenizers 0.22.2", "yoke 0.8.3", "zerocopy", @@ -494,7 +494,7 @@ dependencies = [ "objc2-foundation", "objc2-metal", "once_cell", - "thiserror 2.0.20", + "thiserror 2.0.21", "tracing", ] @@ -514,7 +514,7 @@ dependencies = [ "rayon", "safetensors 0.8.0", "serde", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -545,9 +545,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.7" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54413ede23c2daf518f35156dfde027feb2374004d63bd497f983c8db9c0e313" +checksum = "207aedd8d29a0eb804986bc99eed0d43719fec84a9b7f9c8b70cf83b97cffcc1" dependencies = [ "find-msvc-tools", "jobserver", @@ -953,7 +953,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.9", - "thiserror 2.0.20", + "thiserror 2.0.21", "walkdir", "which", ] @@ -970,9 +970,9 @@ dependencies = [ [[package]] name = "cudarc" -version = "0.19.9" +version = "0.19.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "804764d10e844da09765a7b2ca9641a0851523d1702efb0d7299d73e31b86e80" +checksum = "7359bf1de037ddada72729c3acec9b040f3468400c6e9e740b01ba0cd2e89562" dependencies = [ "float8", "half", @@ -1204,13 +1204,12 @@ checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" [[package]] name = "encoding_rs" -version = "0.8.41" +version = "0.8.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5ef0006ac9ab233c38522f5ae99cae3625151de8f706cacee1cba4b8e2832a" +checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679" dependencies = [ "cfg-if", "core_detect", - "multiversion", "multiversion_no_op", "rustversion", "scopeguard", @@ -1265,9 +1264,9 @@ checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "find-msvc-tools" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef25905e51abafe4dcea6c15fec58c57b601cdbd0ee53d22ea1d3016c587d39b" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" [[package]] name = "flate2" @@ -1880,7 +1879,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.11.0", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-retry", "tokio-util", @@ -1902,7 +1901,7 @@ dependencies = [ "more-asserts", "serde", "serde_json", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-util", "tokio_with_wasm", @@ -2004,9 +2003,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.9" +version = "0.27.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" dependencies = [ "http", "hyper", @@ -2019,16 +2018,17 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", "futures-channel", "futures-util", "http", "http-body", + "httparse", "hyper", "ipnet", "libc", @@ -2284,7 +2284,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.20", + "thiserror 2.0.21", "walkdir", "windows-link", ] @@ -2333,9 +2333,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.105" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" dependencies = [ "cfg-if", "futures-util", @@ -2399,9 +2399,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" -version = "0.1.24" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6480ccc157a1389bb2e4891b24751b0f798ba640d22386f23143fbcc89da195a" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" dependencies = [ "libc", ] @@ -2605,27 +2605,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fafa6961cabd9c63bcd77a45d7e3b7f3b552b70417831fb0f56db717e72407e" -[[package]] -name = "multiversion" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ca4bea16ffc3f443cf7d866912118196bfef4c6a1556ca00f9f9b00bb43f7c" -dependencies = [ - "multiversion-macros", -] - -[[package]] -name = "multiversion-macros" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d416831a7317ef4b08bee00b69cbbb9c8763da7959a7026244d6266869f9c83" -dependencies = [ - "proc-macro2", - "quote", - "rustversion", - "syn 3.0.6", -] - [[package]] name = "multiversion_no_op" version = "1.0.0" @@ -3105,7 +3084,7 @@ dependencies = [ "log", "mem_dbg", "prefetch-index", - "rand 0.10.2", + "rand 0.10.3", "rand_chacha 0.10.0", "rayon", "rdst", @@ -3164,7 +3143,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tracing", "web-time", @@ -3180,14 +3159,14 @@ dependencies = [ "bytes", "getrandom 0.4.3", "lru-slab", - "rand 0.10.2", + "rand 0.10.3", "rand_pcg", "ring", "rustc-hash", "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.20", + "thiserror 2.0.21", "tinyvec", "tracing", "web-time", @@ -3246,9 +3225,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" dependencies = [ "chacha20", "getrandom 0.4.3", @@ -3386,7 +3365,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60dc65c0ff1a7ae1294b0c67b9f14baf70b644404010370171787bfac1038fc0" dependencies = [ "libredox", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -3473,7 +3452,7 @@ dependencies = [ "async-trait", "http", "reqwest", - "thiserror 2.0.20", + "thiserror 2.0.21", "tower-service", ] @@ -3592,9 +3571,9 @@ dependencies = [ [[package]] name = "rustls-platform-verifier" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" dependencies = [ "core-foundation 0.10.1", "core-foundation-sys", @@ -3613,9 +3592,9 @@ dependencies = [ [[package]] name = "rustls-platform-verifier-android" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" +checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" [[package]] name = "rustls-webpki" @@ -3910,9 +3889,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.16.1" +version = "1.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" +checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" [[package]] name = "socket2" @@ -4031,7 +4010,7 @@ dependencies = [ [[package]] name = "sys1" -version = "0.0.2" +version = "0.0.3" dependencies = [ "anyhow", "axum", @@ -4132,11 +4111,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.20" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" dependencies = [ - "thiserror-impl 2.0.20", + "thiserror-impl 2.0.21", ] [[package]] @@ -4152,9 +4131,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.20" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", @@ -4254,7 +4233,7 @@ checksum = "8ae9aa847b762e874fe27be83b120f63fc76fd7f555d254540d16e044660dfeb" dependencies = [ "bitcannon", "serde_json", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -4273,11 +4252,11 @@ dependencies = [ "memchr", "paste", "ptr_hash", - "rand 0.10.2", + "rand 0.10.3", "rayon", "rayon-cond", "spm_precompiled", - "thiserror 2.0.20", + "thiserror 2.0.21", "unicode-normalization", "unicode-normalization-alignments", "unicode-segmentation", @@ -4325,7 +4304,7 @@ dependencies = [ "serde", "serde_json", "spm_precompiled", - "thiserror 2.0.20", + "thiserror 2.0.21", "unicode-normalization-alignments", "unicode-segmentation", "unicode_categories", @@ -4376,7 +4355,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a129d95275ebf4c493ec53bf0f8cd95f5ac161bc4f381700809a54f595d4470" dependencies = [ "pin-project-lite", - "rand 0.10.2", + "rand 0.10.3", "tokio", ] @@ -4494,7 +4473,7 @@ checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" dependencies = [ "crossbeam-channel", "symlink", - "thiserror 2.0.20", + "thiserror 2.0.21", "time", "tracing-subscriber", ] @@ -4871,9 +4850,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.128" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" dependencies = [ "cfg-if", "once_cell", @@ -4884,19 +4863,20 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.78" +version = "0.4.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928" +checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" dependencies = [ "js-sys", + "tokio", "wasm-bindgen", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.128" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4904,9 +4884,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.128" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" dependencies = [ "bumpalo", "proc-macro2", @@ -4917,9 +4897,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.128" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" dependencies = [ "unicode-ident", ] @@ -4939,9 +4919,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.105" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" +checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" dependencies = [ "js-sys", "wasm-bindgen", @@ -5277,7 +5257,7 @@ dependencies = [ "http", "hyper", "more-asserts", - "rand 0.10.2", + "rand 0.10.3", "redb", "reqwest", "reqwest-middleware", @@ -5286,7 +5266,7 @@ dependencies = [ "serde_repr", "statrs", "tempfile", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-retry", "tokio_with_wasm", @@ -5317,12 +5297,12 @@ dependencies = [ "itertools 0.14.0", "lz4_flex", "more-asserts", - "rand 0.10.2", + "rand 0.10.3", "regex", "safe-transmute", "serde", "static_assertions", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-util", "tracing", @@ -5345,12 +5325,12 @@ dependencies = [ "http", "itertools 0.14.0", "more-asserts", - "rand 0.10.2", + "rand 0.10.3", "serde", "serde_json", "sha2 0.11.0", "tempfile", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-util", "tokio_with_wasm", @@ -5385,13 +5365,13 @@ dependencies = [ "more-asserts", "oneshot", "pin-project", - "rand 0.10.2", + "rand 0.10.3", "reqwest", "serde", "serde_json", "shellexpand", "sysinfo", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-util", "tokio_with_wasm", @@ -5464,18 +5444,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.57" +version = "0.8.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873" +checksum = "c17e8fafad82b542ff3717217ecdc736231b59e387768c9630123b4ce4d2db44" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.57" +version = "0.8.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc" +checksum = "595f56e044df4f46a0c9a626f65c3d99eb8488f7e8a8baa12dd76326d9710bf2" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index bd29c64..32a0d2f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "sys1" -version = "0.0.2" +version = "0.0.3" edition = "2024" -description = "System One compatible API for open decision models, e.g. Laya, written in Rust." +description = "System One compatible API for open decision models, written in Rust." authors = ["Alvaro Bartolome <36760800+alvarobartt@users.noreply.github.com>"] default-run = "sys1" license = "Apache-2.0" diff --git a/README.md b/README.md index a02b06d..18145b7 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ - Dynamic, token-based batching - SDPA on CPU, Metal, and CUDA - Flash Attention on Ampere, Ada Lovelace, and Hopper +- Blazing fast inference for Laya ## Get started diff --git a/src/models/laya.rs b/src/models/laya.rs index 751d025..48fe721 100644 --- a/src/models/laya.rs +++ b/src/models/laya.rs @@ -1,8 +1,8 @@ +use super::AttentionImplementation; +use super::DecisionModel; use super::modernbert::{ AttentionOptions, Config as ModernBertConfig, Encoder as ModernBertEncoder, }; -use super::AttentionImplementation; -use super::DecisionModel; use crate::{ device, schema::{ApiError, DecisionRequest, DecisionResponse, Usage}, @@ -10,10 +10,10 @@ use crate::{ }; use anyhow::Context; -use candle_core::{DType, Device, IndexOp, Tensor, D}; -use candle_nn::{embedding, layer_norm, Embedding, LayerNorm, Linear, VarBuilder}; +use candle_core::{D, DType, Device, IndexOp, Tensor}; +use candle_nn::{Embedding, LayerNorm, Linear, VarBuilder, embedding, layer_norm}; use serde::Deserialize; -use serde_json::{json, Map, Value}; +use serde_json::{Map, Value, json}; use std::{collections::HashMap, fs, path::Path}; const TYPES: [&str; 3] = ["choice", "score", "noul"]; @@ -1025,12 +1025,14 @@ mod tests { #[test] fn rejects_flash_attention_without_a_compatible_backend() { - assert!(validate_attention( - AttentionImplementation::FlashAttention2, - &Device::Cpu, - DType::BF16, - ) - .is_err()); + assert!( + validate_attention( + AttentionImplementation::FlashAttention2, + &Device::Cpu, + DType::BF16, + ) + .is_err() + ); validate_attention(AttentionImplementation::Eager, &Device::Cpu, DType::F32).unwrap(); } diff --git a/src/models/modernbert.rs b/src/models/modernbert.rs index 026a980..90089ee 100644 --- a/src/models/modernbert.rs +++ b/src/models/modernbert.rs @@ -1,7 +1,7 @@ use super::AttentionImplementation; -use candle_core::{DType, Device, Result, Tensor, D}; +use candle_core::{D, DType, Device, Result, Tensor}; use candle_nn::{ - embedding, layer_norm_no_bias, ops::softmax, Embedding, LayerNorm, Linear, Module, VarBuilder, + Embedding, LayerNorm, Linear, Module, VarBuilder, embedding, layer_norm_no_bias, ops::softmax, }; use serde::Deserialize; use std::{