expose openvino performance hint and stream count on serve

src/main.cpp

@@ -14,13 +14,17 @@
 //                  [--threads N (all logical CPUs)] [--ctx 8192]
 //                  [--dynamic-quantization 128] (activations in INT8 groups of N values; 0 = f32: slower, and
 //                                    answers no longer move in their last digits with how a call is composed)
+//                  [--hint latency|throughput (latency)] [--streams N (1)]  (OpenVINO's performance hint and
+//                                    inference stream count: streams share a call's work, more decisions a
+//                                    second under concurrent load, slower single requests)
 //
 // JEV_API_KEY set: `jev serve` requires `Authorization: Bearer <key>` on every call but /health.
 // /health reports the SHA-256 of the model folder's files and their fingerprint (model_files, below).
 // DIR holds openvino_model.xml/.bin, tokenizer.gguf (the model's vocabulary) and model.json (its name),
 // written by export/export_openvino.py.
 
 #include <algorithm>
+#include <cctype>
 #include <chrono>
 #include <cstdio>
 #include <cstdlib>
@@ -101,8 +105,8 @@ int main(int argc, char** argv) try {
     const char* usage = "usage: jev serve [options] | jev decide REQUEST.json [--output FILE] [options] (options: top of src/main.cpp)\n";
     if (argc < 2 || (std::string(argv[1]) != "serve" && std::string(argv[1]) != "decide")) { std::fputs(usage, stderr); return 2; }
     const std::string command = argv[1];
-    std::string host = "127.0.0.1", name, input, output;
-    int port = 8017, threads = static_cast<int>(std::max(1u, std::thread::hardware_concurrency()));
+    std::string host = "127.0.0.1", name, input, output, hint = "LATENCY";
+    int port = 8017, threads = static_cast<int>(std::max(1u, std::thread::hardware_concurrency())), streams = 1;
     // batch_tokens: requests read together pay off while they are small (measured, one call vs separate
     // calls: 4 x 64 tokens -11%, 3 x 120 -16%, 2 x 150 +3%, 4 x 200 +17%): the matmuls gain rows, the
     // attention of every block runs over all the call's cells.
@@ -119,6 +123,9 @@ int main(int argc, char** argv) try {
         else if (a == "--threads") threads = std::stoi(next());
         else if (a == "--ctx") ctx = std::stoul(next());
         else if (a == "--dynamic-quantization") quantization_group = std::stoul(next());
+        else if (a == "--hint") { hint = next(); std::transform(hint.begin(), hint.end(), hint.begin(), [](unsigned char c) { return static_cast<char>(std::toupper(c)); });
+                                  if (hint != "LATENCY" && hint != "THROUGHPUT") throw std::runtime_error(a + " is latency or throughput"); }
+        else if (a == "--streams") streams = std::stoi(next());
         else if (serving && a == "--host") host = next();
         else if (serving && a == "--port") port = std::stoi(next());
         else if (serving && a == "--warmup") warmup = std::stoul(next());
@@ -157,7 +164,7 @@ int main(int argc, char** argv) try {
     if (command == "serve") files = std::async(std::launch::async, model_files, model_dir);
     Vocab vocab(model_dir / "tokenizer.gguf");
     std::fprintf(stderr, "jev: loading %s (OpenVINO %s)\n", model_dir.string().c_str(), Model::openvino_version().c_str());
-    Model model(model_dir, threads, command == "serve" ? state_cache : 0, state_cache_tokens, quantization_group);
+    Model model(model_dir, threads, hint, streams, command == "serve" ? state_cache : 0, state_cache_tokens, quantization_group);
     if (command == "decide") {
         Scheduler scheduler(model, 0, 0);
         Api api{model, vocab, scheduler, name, ctx, ojson::object()};

src/model.cpp

@@ -53,11 +53,13 @@ struct Model::Impl {
     std::vector<bool> seen;  // call lengths T already run: their kernels exist
     std::atomic<size_t> batched{0};
     size_t quantization_group;
+    std::string hint;
+    int streams;
     ov::CompiledModel compiled;
     ov::InferRequest req;
 
-    Impl(const fs::path& d, int threads, size_t snapshots, size_t snapshot_tokens, size_t quantization_group)
-        : dir(d), snaps(snapshots, snapshot_tokens), seen(PIECE_TOKENS + 1, false), quantization_group(quantization_group) {
+    Impl(const fs::path& d, int threads, const std::string& hint, int streams, size_t snapshots, size_t snapshot_tokens, size_t quantization_group)
+        : dir(d), snaps(snapshots, snapshot_tokens), seen(PIECE_TOKENS + 1, false), quantization_group(quantization_group), hint(hint), streams(streams) {
         auto graph = core.read_model((dir / "openvino_model.xml").string());
         std::set<std::string> names;
         for (auto& in : graph->inputs())
@@ -83,7 +85,10 @@ struct Model::Impl {
             head_dim = static_cast<size_t>(ps[3].get_length());
         }
         for (auto& n : kv_names) present_names.push_back("present_" + n.substr(5));  // past_key.N -> present_key.N
-        ov::AnyMap cfg = {{"PERFORMANCE_HINT", "LATENCY"}, {"NUM_STREAMS", "1"}, {"INFERENCE_NUM_THREADS", threads},
+        // PERFORMANCE_HINT/NUM_STREAMS are the released defaults (--hint latency, --streams 1); throughput
+        // and streams > 1 share a call's work: more decisions a second under concurrent load, slower single
+        // requests.
+        ov::AnyMap cfg = {{"PERFORMANCE_HINT", hint}, {"NUM_STREAMS", std::to_string(streams)}, {"INFERENCE_NUM_THREADS", threads},
                           // Activations quantized to INT8 in groups of quantization_group values (0: kept f32).
                           // 128 rather than 32: -17% on one question, the same accuracy on the 999 set (0.758 vs
                           // 0.759; mean |dP| 0.008). 256 saves 3% more, max |dP| 0.29.
@@ -313,8 +318,8 @@ struct Model::Impl {
     }
 };
 
-Model::Model(const fs::path& dir, int threads, size_t snapshots, size_t snapshot_tokens, size_t quantization_group)
-    : impl_(std::make_unique<Impl>(dir, threads, snapshots, snapshot_tokens, quantization_group)) {}
+Model::Model(const fs::path& dir, int threads, const std::string& hint, int streams, size_t snapshots, size_t snapshot_tokens, size_t quantization_group)
+    : impl_(std::make_unique<Impl>(dir, threads, hint, streams, snapshots, snapshot_tokens, quantization_group)) {}
 Model::~Model() = default;
 
 std::vector<std::vector<double>> Model::score_batch(const std::vector<const ScoreRequest*>& reqs) { return impl_->score_batch(reqs); }
@@ -334,7 +339,7 @@ void Model::warmup() {
 ojson Model::describe() const {
     const Impl& m = *impl_;
     ojson e = {{"runtime", "openvino"}, {"version", openvino_version()}, {"device", "cpu"}, {"model_dir", m.dir.string()},
-               {"dynamic_quantization", m.quantization_group}};
+               {"dynamic_quantization", m.quantization_group}, {"performance_hint", m.hint}, {"streams", m.streams}};
     e["state_snapshots"] = m.snaps.stats();
     e["state_snapshots"]["bytes"] = m.snaps.kept_tokens() * m.kv_names.size() * m.kv_heads * m.head_dim * sizeof(float);
     e["batched_requests"] = m.batched.load();

src/model.hpp

@@ -31,8 +31,10 @@ class Model {
     // `dir` holds openvino_model.xml/.bin; `snapshots` states of at most `snapshot_tokens` tokens in all
     // are kept for later requests (0 = none). Activations are quantized to INT8 in groups of
     // `quantization_group` values (0 = kept f32: slower, and an answer no longer moves, in its last digits,
-    // with how a call is composed).
-    Model(const std::filesystem::path& dir, int threads, size_t snapshots, size_t snapshot_tokens, size_t quantization_group);
+    // with how a call is composed). `hint` (LATENCY/THROUGHPUT) and `streams` are OpenVINO's performance
+    // hint and inference stream count: streams share a call's work, more decisions a second under
+    // concurrent load, slower single requests.
+    Model(const std::filesystem::path& dir, int threads, const std::string& hint, int streams, size_t snapshots, size_t snapshot_tokens, size_t quantization_group);
     ~Model();
     Model(const Model&) = delete;
     Model& operator=(const Model&) = delete;