diff --git a/src/main.cpp b/src/main.cpp index 8a52f5c..bd878e2 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -14,6 +14,9 @@ // [--threads N (all logical CPUs)] [--ctx 8192] // [--dynamic-quantization 128] (activations in INT8 groups of N values; 0 = f32: slower, and // answers no longer move in their last digits with how a call is composed) +// [--hint latency|throughput (latency)] [--streams N (1)] (OpenVINO's performance hint and +// inference stream count: streams share a call's work, more decisions a +// second under concurrent load, slower single requests) // // JEV_API_KEY set: `jev serve` requires `Authorization: Bearer ` on every call but /health. // /health reports the SHA-256 of the model folder's files and their fingerprint (model_files, below). @@ -21,6 +24,7 @@ // written by export/export_openvino.py. #include +#include #include #include #include @@ -101,8 +105,8 @@ int main(int argc, char** argv) try { const char* usage = "usage: jev serve [options] | jev decide REQUEST.json [--output FILE] [options] (options: top of src/main.cpp)\n"; if (argc < 2 || (std::string(argv[1]) != "serve" && std::string(argv[1]) != "decide")) { std::fputs(usage, stderr); return 2; } const std::string command = argv[1]; - std::string host = "127.0.0.1", name, input, output; - int port = 8017, threads = static_cast(std::max(1u, std::thread::hardware_concurrency())); + std::string host = "127.0.0.1", name, input, output, hint = "LATENCY"; + int port = 8017, threads = static_cast(std::max(1u, std::thread::hardware_concurrency())), streams = 1; // batch_tokens: requests read together pay off while they are small (measured, one call vs separate // calls: 4 x 64 tokens -11%, 3 x 120 -16%, 2 x 150 +3%, 4 x 200 +17%): the matmuls gain rows, the // attention of every block runs over all the call's cells. @@ -119,6 +123,9 @@ int main(int argc, char** argv) try { else if (a == "--threads") threads = std::stoi(next()); else if (a == "--ctx") ctx = std::stoul(next()); else if (a == "--dynamic-quantization") quantization_group = std::stoul(next()); + else if (a == "--hint") { hint = next(); std::transform(hint.begin(), hint.end(), hint.begin(), [](unsigned char c) { return static_cast(std::toupper(c)); }); + if (hint != "LATENCY" && hint != "THROUGHPUT") throw std::runtime_error(a + " is latency or throughput"); } + else if (a == "--streams") streams = std::stoi(next()); else if (serving && a == "--host") host = next(); else if (serving && a == "--port") port = std::stoi(next()); else if (serving && a == "--warmup") warmup = std::stoul(next()); @@ -157,7 +164,7 @@ int main(int argc, char** argv) try { if (command == "serve") files = std::async(std::launch::async, model_files, model_dir); Vocab vocab(model_dir / "tokenizer.gguf"); std::fprintf(stderr, "jev: loading %s (OpenVINO %s)\n", model_dir.string().c_str(), Model::openvino_version().c_str()); - Model model(model_dir, threads, command == "serve" ? state_cache : 0, state_cache_tokens, quantization_group); + Model model(model_dir, threads, hint, streams, command == "serve" ? state_cache : 0, state_cache_tokens, quantization_group); if (command == "decide") { Scheduler scheduler(model, 0, 0); Api api{model, vocab, scheduler, name, ctx, ojson::object()}; diff --git a/src/model.cpp b/src/model.cpp index e5340e5..8b3e54c 100644 --- a/src/model.cpp +++ b/src/model.cpp @@ -53,11 +53,13 @@ struct Model::Impl { std::vector seen; // call lengths T already run: their kernels exist std::atomic batched{0}; size_t quantization_group; + std::string hint; + int streams; ov::CompiledModel compiled; ov::InferRequest req; - Impl(const fs::path& d, int threads, size_t snapshots, size_t snapshot_tokens, size_t quantization_group) - : dir(d), snaps(snapshots, snapshot_tokens), seen(PIECE_TOKENS + 1, false), quantization_group(quantization_group) { + Impl(const fs::path& d, int threads, const std::string& hint, int streams, size_t snapshots, size_t snapshot_tokens, size_t quantization_group) + : dir(d), snaps(snapshots, snapshot_tokens), seen(PIECE_TOKENS + 1, false), quantization_group(quantization_group), hint(hint), streams(streams) { auto graph = core.read_model((dir / "openvino_model.xml").string()); std::set names; for (auto& in : graph->inputs()) @@ -83,7 +85,10 @@ struct Model::Impl { head_dim = static_cast(ps[3].get_length()); } for (auto& n : kv_names) present_names.push_back("present_" + n.substr(5)); // past_key.N -> present_key.N - ov::AnyMap cfg = {{"PERFORMANCE_HINT", "LATENCY"}, {"NUM_STREAMS", "1"}, {"INFERENCE_NUM_THREADS", threads}, + // PERFORMANCE_HINT/NUM_STREAMS are the released defaults (--hint latency, --streams 1); throughput + // and streams > 1 share a call's work: more decisions a second under concurrent load, slower single + // requests. + ov::AnyMap cfg = {{"PERFORMANCE_HINT", hint}, {"NUM_STREAMS", std::to_string(streams)}, {"INFERENCE_NUM_THREADS", threads}, // Activations quantized to INT8 in groups of quantization_group values (0: kept f32). // 128 rather than 32: -17% on one question, the same accuracy on the 999 set (0.758 vs // 0.759; mean |dP| 0.008). 256 saves 3% more, max |dP| 0.29. @@ -313,8 +318,8 @@ struct Model::Impl { } }; -Model::Model(const fs::path& dir, int threads, size_t snapshots, size_t snapshot_tokens, size_t quantization_group) - : impl_(std::make_unique(dir, threads, snapshots, snapshot_tokens, quantization_group)) {} +Model::Model(const fs::path& dir, int threads, const std::string& hint, int streams, size_t snapshots, size_t snapshot_tokens, size_t quantization_group) + : impl_(std::make_unique(dir, threads, hint, streams, snapshots, snapshot_tokens, quantization_group)) {} Model::~Model() = default; std::vector> Model::score_batch(const std::vector& reqs) { return impl_->score_batch(reqs); } @@ -334,7 +339,7 @@ void Model::warmup() { ojson Model::describe() const { const Impl& m = *impl_; ojson e = {{"runtime", "openvino"}, {"version", openvino_version()}, {"device", "cpu"}, {"model_dir", m.dir.string()}, - {"dynamic_quantization", m.quantization_group}}; + {"dynamic_quantization", m.quantization_group}, {"performance_hint", m.hint}, {"streams", m.streams}}; e["state_snapshots"] = m.snaps.stats(); e["state_snapshots"]["bytes"] = m.snaps.kept_tokens() * m.kv_names.size() * m.kv_heads * m.head_dim * sizeof(float); e["batched_requests"] = m.batched.load(); diff --git a/src/model.hpp b/src/model.hpp index feb2f4c..726bcf9 100644 --- a/src/model.hpp +++ b/src/model.hpp @@ -31,8 +31,10 @@ class Model { // `dir` holds openvino_model.xml/.bin; `snapshots` states of at most `snapshot_tokens` tokens in all // are kept for later requests (0 = none). Activations are quantized to INT8 in groups of // `quantization_group` values (0 = kept f32: slower, and an answer no longer moves, in its last digits, - // with how a call is composed). - Model(const std::filesystem::path& dir, int threads, size_t snapshots, size_t snapshot_tokens, size_t quantization_group); + // with how a call is composed). `hint` (LATENCY/THROUGHPUT) and `streams` are OpenVINO's performance + // hint and inference stream count: streams share a call's work, more decisions a second under + // concurrent load, slower single requests. + Model(const std::filesystem::path& dir, int threads, const std::string& hint, int streams, size_t snapshots, size_t snapshot_tokens, size_t quantization_group); ~Model(); Model(const Model&) = delete; Model& operator=(const Model&) = delete;