From dff25e07c59ed470aa2e32ad1822e8dd900cf9e2 Mon Sep 17 00:00:00 2001 From: Paul-Louis NECH Date: Wed, 14 Oct 2026 10:55:10 +0200 Subject: [PATCH] contrib: GPU serving via llama.cpp (13-19x latency, 89% agreement) Optional backend: a ~260-line proxy speaking jev's /v1/systemone wire format, answers computed by llama.cpp decoding a quantized GGUF instead of the OpenVINO graph. Reuses jev's own wire/prompt/vocab/choice/score sources unchanged; one decode per question on the shared state prefix with KV rolled back in between; P(yes) from raw logits. Receipts (RTX 4080 Laptop, q4_k_m, llama.cpp b11541): - 1-question p50 22 ms vs 292 ms CPU (13x) - ~300-token state, 5 questions: 73 ms vs 1372 ms (19x) - tokenizer parity exact (usage.input_tokens match on every request) - 89% overall decision agreement vs jev serve (100% danger/dodge, 95-100% fire/spatial, mean |dP| 0.02-0.16); results.example.json Includes validate.py (seeded agreement + latency vs jev serve) and a Makefile pinning llama.cpp b11541. Caveats in README: single client, no KV snapshots across requests, cold-vs-warm GPU clocks. --- contrib/gpu-llama/Makefile | 16 ++ contrib/gpu-llama/README.md | 68 +++++++ contrib/gpu-llama/proxy.cpp | 263 +++++++++++++++++++++++++ contrib/gpu-llama/results.example.json | 112 +++++++++++ contrib/gpu-llama/validate.py | 214 ++++++++++++++++++++ 5 files changed, 673 insertions(+) create mode 100644 contrib/gpu-llama/Makefile create mode 100644 contrib/gpu-llama/README.md create mode 100644 contrib/gpu-llama/proxy.cpp create mode 100644 contrib/gpu-llama/results.example.json create mode 100644 contrib/gpu-llama/validate.py diff --git a/contrib/gpu-llama/Makefile b/contrib/gpu-llama/Makefile new file mode 100644 index 0000000..a1d97ca --- /dev/null +++ b/contrib/gpu-llama/Makefile @@ -0,0 +1,16 @@ +# llama.cpp CUDA build prefix: must contain include/llama.h (or include/ggml/) and lib/ +LLAMA_DIR ?= /usr/local +CXX ?= g++ +CXXFLAGS ?= -std=c++17 -O3 + +jev-gpu: proxy.cpp + $(CXX) $(CXXFLAGS) -o $@ proxy.cpp \ + ../../src/wire.cpp ../../src/vocab.cpp ../../src/request_tokens.cpp \ + -I ../../src -I ../../third_party \ + -I $(LLAMA_DIR)/include -I $(LLAMA_DIR)/include/ggml \ + -pthread -L $(LLAMA_DIR)/lib -lllama -lggml -Wl,-rpath,$(LLAMA_DIR)/lib + +clean: + rm -f jev-gpu + +.PHONY: clean diff --git a/contrib/gpu-llama/README.md b/contrib/gpu-llama/README.md new file mode 100644 index 0000000..98307f1 --- /dev/null +++ b/contrib/gpu-llama/README.md @@ -0,0 +1,68 @@ +# GPU serving via llama.cpp + +An optional GPU backend for `jev serve`: the same `/v1/systemone` wire format, answers +computed by llama.cpp decoding a quantized GGUF instead of the OpenVINO graph. Built and +battle-tested as the default lane of a real-time demo (jev judging 10-15x/s in a browser +game loop, official 5x90s validation runs on it). + +- `proxy.cpp` (~260 lines): HTTP server speaking jev's wire format. Reuses jev's own + `wire`/`prompt`/`vocab`/`choice`/`score`/`request_tokens` sources unchanged; only the + scoring backend differs. One llama.cpp decode per question, on top of the shared state + prefix, with the KV cache rolled back in between — every question sees the state alone, + the way jev's attention mask does. P(yes) is read from the raw logits: + `sigmoid(logit("1") - logit("0"))`. (In the released weights the answer head is + half-zero by design — `row("0") = 0` — so this equals `sigmoid(logit("1"))`.) +- `validate.py`: sends the same seeded scenario set to `jev serve` (CPU, port 8017) and + the GPU proxy (port 8153), reports per-question decision agreement + mean |dP|, and + measures p50/p95 for a short 1-question and a long 5-question request. Writes `results.json`. + +## Receipts (RTX 4080 Laptop GPU, q4_k_m GGUF, llama.cpp b11541) + +| metric | CPU `jev serve` | GPU lane | ratio | +|---|---|---|---| +| 1-question p50 | 292 ms | 22 ms | 13x | +| ~300-token state, 5 questions, p50 | 1372 ms | 73 ms | 19x | +| tokenizer parity (`usage.input_tokens`) | — | exact match on every request checked | — | +| decision agreement vs `jev serve` | — | 89% overall; 100% danger/dodge, 95-100% fire/spatial; mean \|dP\| 0.02-0.16 | — | + +## Build + +Get llama.cpp (pinned `b11541`, CUDA build) and the GGUF: + +```bash +git clone --depth 1 --branch b11541 https://github.com/ggml-org/llama.cpp +cd llama.cpp && cmake -B build -DGGML_CUDA=ON && cmake --build build --config Release -j +# the GGUF is on Hugging Face: https://huggingface.co/feder-cr/jevos-v4 (q4_k_m quant) +``` + +Then, from `contrib/gpu-llama/`: + +```bash +make LLAMA_DIR=/path/to/llama.cpp/build +# -> ./jev-gpu +``` + +## Run + +```bash +./jev-gpu --model /path/to/jevos-v4-q4_k_m.gguf --port 8153 +curl -s http://127.0.0.1:8153/v1/systemone -H 'Content-Type: application/json' -d '{ + "model":"jev-latest","state":"No bombs are in the air.", + "questions":{"q":{"type":"noul","instructions":"Is a bomb dangerously close to the ship?"}}}' +``` + +Validate agreement + latency against a running `jev serve` on 8017: + +```bash +python3 validate.py # writes results.json +``` + +## Caveats + +- **One client at a time.** Requests serialize on a mutex; a second concurrent client + turns a 42 ms p50 into ~487 ms. This matches jev's `NUM_STREAMS=1` behavior on OpenVINO. +- No KV snapshots across requests: every request re-reads its state. (jev's OpenVINO + server keeps state snapshots; that behavior is not replicated here.) +- Idle-GPU clocks: a cold GPU sits at ~210 MHz, and the first requests are 60% slower + than warm steady state. Warm up before benchmarking. +- Quantization: numbers above are q4_k_m. Higher quants trade latency for agreement. diff --git a/contrib/gpu-llama/proxy.cpp b/contrib/gpu-llama/proxy.cpp new file mode 100644 index 0000000..1d8f97d --- /dev/null +++ b/contrib/gpu-llama/proxy.cpp @@ -0,0 +1,263 @@ +// jevos-v4 on the GPU: jev's /v1/systemone wire format served from a GGUF through llama.cpp. +// +// The wire layer (validation, prompt rendering, tokenization, answer math, response shape, usage) +// is jev's own code: wire.cpp, prompt.hpp, request_tokens.cpp, choice.hpp, score.hpp, pyjson.hpp. +// Only the scoring backend differs from jev: instead of OpenVINO's answer-head graph, each binary +// job is one llama.cpp decode of the state prefix plus the job's own tokens, and P(yes) is read +// from the logits of the answer tokens "0" and "1" at the last position: +// P = sigmoid(logit("1") - logit("0")) +// exactly as jev's served OpenVINO graph does with its 2-row answer head. +#include +#include +#include +#include +#include +#include +#include +#include + +#include "httplib.h" + +#include "errors.hpp" +#include "pyjson.hpp" +#include "text.hpp" +#include "prompt.hpp" +#include "wire.hpp" +#include "choice.hpp" +#include "score.hpp" +#include "vocab.hpp" +#include "request_tokens.hpp" + +#include "llama.h" + +static const size_t CTX = 8192; +static const char* SERVED = "jevos-v4"; +static const char* ALIAS_PREFIX = "jev-"; + +// ---------------------------------------------------------------- llama scoring + +struct LlamaScorer { + llama_model* model = nullptr; + llama_context* ctx = nullptr; + const llama_vocab* vocab_ll = nullptr; + llama_token t0 = -1, t1 = -1; // the answer tokens "0" and "1" + std::mutex mu; // one context: serve requests one at a time + + LlamaScorer(const std::string& gguf_path) { + llama_backend_init(); + llama_model_params mp = llama_model_default_params(); + mp.n_gpu_layers = 99; + model = llama_model_load_from_file(gguf_path.c_str(), mp); + if (!model) throw std::runtime_error("cannot load " + gguf_path); + vocab_ll = llama_model_get_vocab(model); + llama_context_params cp = llama_context_default_params(); + cp.n_ctx = CTX; + cp.n_batch = CTX; + cp.n_ubatch = 2048; + ctx = llama_init_from_model(model, cp); + if (!ctx) throw std::runtime_error("cannot create the llama context"); + // the answer tokens must each be a single token, as the OpenVINO export asserts + for (const char* d : {"0", "1"}) { + llama_token tmp[4]; + int k = llama_tokenize(vocab_ll, d, 1, tmp, 4, false, true); + if (k != 1) throw std::runtime_error(std::string("the answer digit '") + d + "' is not one token"); + if (d[0] == '0') t0 = tmp[0]; else t1 = tmp[0]; + } + } + + ~LlamaScorer() { + if (ctx) llama_free(ctx); + if (model) llama_model_free(model); + } + + // P(yes) for every job of one request: the shared prefix is decoded once, each job then reads + // its own tokens on top of it (llama.cpp rolls the cache back, so every job sees the state and + // itself alone, as jev's attention mask does). + std::vector score(const RequestTokens& rt) { + std::lock_guard lk(mu); + llama_memory_t mem = llama_get_memory(ctx); + llama_memory_clear(mem, true); + std::vector ps(rt.prompts.size()); + if (!rt.prefix.empty()) { + llama_batch b = llama_batch_get_one(const_cast(rt.prefix.data()), (int) rt.prefix.size()); + if (llama_decode(ctx, b)) throw std::runtime_error("llama_decode failed on the shared prefix"); + } + for (size_t i = 0; i < rt.prompts.size(); ++i) { + const Tokens& t = rt.prompts[i]; + const llama_token* suffix = t.data() + rt.prefix.size(); + int n = (int) (t.size() - rt.prefix.size()); + llama_batch b = llama_batch_get_one(const_cast(suffix), n); + if (llama_decode(ctx, b)) throw std::runtime_error("llama_decode failed on a question"); + const float* lg = llama_get_logits_ith(ctx, -1); + double z = (double) lg[t1] - (double) lg[t0]; + ps[i] = 1.0 / (1.0 + std::exp(-z)); + if (i + 1 < rt.prompts.size() && !rt.prefix.empty()) + llama_memory_seq_rm(mem, 0, rt.prefix.size(), -1); + } + return ps; + } + + void warmup() { + RequestTokens rt; + Tokens warm = rt.prefix; + std::string text = "warmup"; + Tokens t; + t.resize(text.size() + 8); + int n = llama_tokenize(vocab_ll, text.data(), (int) text.size(), t.data(), (int) t.size(), false, true); + t.resize(n); + rt.prompts.push_back(t); + score(rt); + } +}; + +// ---------------------------------------------------------------- the request pipeline +// Everything below mirrors jev's src/server.cpp systemone(): the same prompt construction, the same +// answer assembly, the same usage count, the same response bytes. + +static void add_prompts(const std::vector& qs, std::vector& question_of, std::vector& instructions) { + for (size_t k = 0; k < qs.size(); ++k) { + if (qs[k].kind == "boolean") { + question_of.push_back(k); + instructions.push_back(qs[k].instructions); + continue; + } + std::vector described; + for (auto& [field, text] : qs[k].texts) described.push_back(text); + if (qs[k].kind == "choice") + for (auto& phrasing : CHOICE_PHRASINGS) + for (size_t i = 0; i < described.size(); ++i) { + question_of.push_back(k); + instructions.push_back(choice_instructions(qs[k].instructions, described, i, phrasing)); + } + else + for (auto& phrasing : SCORE_PHRASINGS) + for (size_t i = 1; i < described.size(); ++i) { + question_of.push_back(k); + instructions.push_back(score_instructions(qs[k].instructions, described, i, phrasing)); + } + } +} + +int main(int argc, char** argv) { + std::string gguf_path, host = "127.0.0.1"; + int port = 8153; + for (int i = 1; i < argc; ++i) { + std::string a = argv[i]; + auto next = [&]() { return std::string(argv[++i]); }; + if (a == "--model") gguf_path = next(); + else if (a == "--port") port = std::stoi(next()); + else if (a == "--host") host = next(); + else { fprintf(stderr, "unknown arg %s\n", a.c_str()); return 1; } + } + if (gguf_path.empty()) { fprintf(stderr, "usage: %s --model MODEL.gguf [--port N] [--host H]\n", argv[0]); return 1; } + + Vocab vocab(gguf_path); // the tokenizer, as jev reads it from the GGUF + LlamaScorer scorer(gguf_path); // the model, on the GPU + scorer.warmup(); + + httplib::Server svr; + svr.new_task_queue = [] { return new httplib::ThreadPool(4); }; + + svr.Get("/health", [](const httplib::Request&, httplib::Response& res) { + res.set_content("{\"status\":\"ok\"}", "application/json"); + }); + + svr.Post("/v1/systemone", [&](const httplib::Request& req, httplib::Response& res) { + Value body; + try { + body = pyjson::loads(req.body); + } catch (const pyjson::DecodeError& e) { + res.status = 422; + res.set_content("{\"error\":[{\"type\":\"json_invalid\",\"loc\":[\"body\"],\"msg\":\"JSON decode error\",\"input\":{}}]}", "application/json"); + return; + } catch (const pyjson::BodyError&) { + res.status = 400; + res.set_content("{\"error\":[{\"type\":\"body_error\",\"loc\":[\"body\"],\"msg\":\"Invalid body\"}]}", "application/json"); + return; + } + Errs errs; + check_wire(body, errs); + if (errs.empty()) { + const Value* model = body.get("model"); + const std::string& requested = model->s; + if (!(requested == SERVED || requested.rfind(ALIAS_PREFIX, 0) == 0)) + errs.push_back(app_err({S("body"), S("model")}, std::string("Unknown model ") + pyjson::str_repr(requested) + ": this server answers as " + SERVED)); + } + if (errs.empty()) { + std::vector qs; + for (auto& [qid, q] : body.get("questions")->dict) qs.push_back(to_native(qid, q)); + check_native(*body.get("state"), qs, errs); + if (errs.empty()) { + try { + auto answered = [&]() -> std::string { + std::vector question_of; + std::vector instructions; + add_prompts(qs, question_of, instructions); + std::string state_text = render_state(*body.get("state")); + for (auto& in : instructions) + if (size_t n = vocab.tokenize(binary_prompt(vocab.bos(), state_text, in)).size(); n > CTX) + throw unprocessable(app_err({S("body")}, "Prompt is " + std::to_string(n) + " tokens, over the context limit " + std::to_string(CTX))); + RequestTokens rt = tokenize_request(vocab, state_text, instructions); + for (auto& t : rt.prompts) + if (t.size() > CTX) throw unprocessable(app_err({S("body")}, "Prompt is " + std::to_string(t.size()) + " tokens, over the context limit " + std::to_string(CTX))); + std::vector ps = scorer.score(rt); + + // usage counts the shared state once, as the Python server does + size_t compiled = 0, shared = rt.prompts.size() > 1 ? rt.state : 0; + for (auto& j : rt.prompts) compiled += j.size(); + + std::vector> answered(qs.size()); + for (size_t i = 0; i < ps.size(); ++i) answered[question_of[i]].push_back(ps[i]); + + std::string out = "{\"model\":"; + pyjson::quote(out, SERVED); + out += ",\"answers\":{"; + for (size_t k = 0; k < qs.size(); ++k) { + if (k) out += ","; + pyjson::quote(out, qs[k].id); + if (qs[k].kind == "boolean") { + out += ":{\"type\":\"noul\",\"noul\":" + pyjson::float_repr(answered[k][0]) + "}"; + continue; + } + if (qs[k].kind == "score") { + const std::vector& legend = qs[k].legend; + ScoreAnswer a = rate(answered[k], legend.size()); + out += ":{\"type\":\"score\",\"score\":" + pyjson::float_repr(a.score) + ",\"legend\":{"; + for (size_t i = 0; i < legend.size(); ++i) out += (i ? ",\"" : "\"") + std::to_string(i) + "\":" + legend[i]; + out += "},\"probabilities\":{"; + for (size_t i = 0; i < legend.size(); ++i) out += (i ? ",\"" : "\"") + std::to_string(i) + "\":" + pyjson::float_repr(a.probabilities[i]); + out += "},\"confidence\":" + pyjson::float_repr(a.confidence) + "}"; + continue; + } + const std::vector& options = qs[k].options; + ChoiceAnswer a = choose(answered[k], options.size()); + out += ":{\"type\":\"choice\",\"choice\":"; + pyjson::quote(out, options[a.best]); + out += ",\"probabilities\":{"; + for (size_t i = 0; i < options.size(); ++i) { + if (i) out += ","; + pyjson::quote(out, options[i]); + out += ":" + pyjson::float_repr(a.probabilities[i]); + } + out += "},\"confidence\":" + pyjson::float_repr(a.confidence) + "}"; + } + out += "},\"usage\":{\"input_tokens\":" + std::to_string(compiled - shared * (rt.prompts.size() - 1)) + ",\"output_tokens\":0}}"; + return out; + }(); + res.set_content(answered, "application/json"); + return; + } catch (const ApiError& e) { + res.status = e.status; + res.set_content(error_body(e), "application/json"); + return; + } + } + } + res.status = 422; + res.set_content(error_body(unprocessable(std::move(errs))), "application/json"); + }); + + fprintf(stderr, "jevos-gpu listening on http://%s:%d (model: %s)\n", host.c_str(), port, gguf_path.c_str()); + if (!svr.listen(host.c_str(), port)) { fprintf(stderr, "cannot bind\n"); return 1; } + return 0; +} diff --git a/contrib/gpu-llama/results.example.json b/contrib/gpu-llama/results.example.json new file mode 100644 index 0000000..cbdc729 --- /dev/null +++ b/contrib/gpu-llama/results.example.json @@ -0,0 +1,112 @@ +{ + "scenarios": 120, + "per_question": { + "danger/hit": { + "n": 120, + "agree": 1.0, + "cpu_acc": 0.4667, + "gpu_acc": 0.4667, + "mean_dp": 0.0289, + "p95_dp": 0.0546, + "max_dp": 0.0612 + }, + "danger/land": { + "n": 120, + "agree": 1.0, + "cpu_acc": 0.4667, + "gpu_acc": 0.4667, + "mean_dp": 0.0181, + "p95_dp": 0.0301, + "max_dp": 0.0319 + }, + "fire/should": { + "n": 120, + "agree": 0.9583, + "cpu_acc": 0.2, + "gpu_acc": 0.225, + "mean_dp": 0.048, + "p95_dp": 0.1144, + "max_dp": 0.1348 + }, + "fire/line": { + "n": 120, + "agree": 0.95, + "cpu_acc": 0.2583, + "gpu_acc": 0.3083, + "mean_dp": 0.097, + "p95_dp": 0.1796, + "max_dp": 0.3132 + }, + "fire/hit": { + "n": 120, + "agree": 0.8, + "cpu_acc": 0.275, + "gpu_acc": 0.2583, + "mean_dp": 0.0822, + "p95_dp": 0.1645, + "max_dp": 0.1938 + }, + "fire/aiming": { + "n": 120, + "agree": 0.6417, + "cpu_acc": 0.275, + "gpu_acc": 0.6, + "mean_dp": 0.1562, + "p95_dp": 0.3109, + "max_dp": 0.508 + }, + "spatial/bombL": { + "n": 120, + "agree": 0.8417, + "cpu_acc": 0.7917, + "gpu_acc": 0.7333, + "mean_dp": 0.0914, + "p95_dp": 0.3335, + "max_dp": 0.4275 + }, + "spatial/alienL": { + "n": 120, + "agree": 0.9, + "cpu_acc": 0.875, + "gpu_acc": 0.825, + "mean_dp": 0.0776, + "p95_dp": 0.3504, + "max_dp": 0.5818 + }, + "dodge/choice": { + "n": 32, + "agree": 1.0 + } + }, + "overall_agree": 0.8901, + "latency": { + "short/1q": { + "cpu": { + "p50_ms": 292.2, + "p95_ms": 437.3, + "mean_ms": 302.4, + "n": 30 + }, + "gpu": { + "p50_ms": 22.1, + "p95_ms": 28.1, + "mean_ms": 22.8, + "n": 30 + } + }, + "long/5q": { + "cpu": { + "p50_ms": 1372.2, + "p95_ms": 2170.3, + "mean_ms": 1600.2, + "n": 30 + }, + "gpu": { + "p50_ms": 72.6, + "p95_ms": 91.4, + "mean_ms": 74.0, + "n": 30 + } + } + } +} \ No newline at end of file diff --git a/contrib/gpu-llama/validate.py b/contrib/gpu-llama/validate.py new file mode 100644 index 0000000..dd8bb22 --- /dev/null +++ b/contrib/gpu-llama/validate.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +"""jevos GPU lane: answer-agreement validation and latency, GPU llama.cpp vs CPU OpenVINO. + +Sends the same seeded scenario set to both servers, compares decisions and probabilities, +then measures p50/p95 latency for a short/1-question and a long/5-question request. +Writes results.json. Rate against the CPU server is capped at ~2 requests/second. +""" +import json +import random +import statistics +import time +import urllib.request +from concurrent.futures import ThreadPoolExecutor + +CPU = "http://127.0.0.1:8017/v1/systemone" +GPU = "http://127.0.0.1:8153/v1/systemone" + +QUESTIONS = { + "danger/hit": {"type": "noul", "instructions": "Is the ship about to be hit by a bomb?"}, + "danger/land": {"type": "noul", "instructions": "Is a bomb about to land on the ship?"}, + "fire/should": {"type": "noul", "instructions": "Should the ship fire its gun right now?"}, + "fire/line": {"type": "noul", "instructions": "Is an alien in the line of fire right now?"}, + "fire/hit": {"type": "noul", "instructions": "Would firing now likely hit an alien?"}, + "fire/aiming": {"type": "noul", "instructions": "Is the ship aiming at an alien right now?"}, + "spatial/bombL": {"type": "noul", "instructions": "Is the nearest bomb to the left of the ship?"}, + "spatial/alienL": {"type": "noul", "instructions": "Is the lowest alien to the left of the ship?"}, + "dodge/choice": {"type": "choice", "instructions": "Which way should the ship move right now?", + "criteria": {"left": "move left", "right": "move right"}}, +} + + +def make_scenarios(n=120, seed=42): + """Same generator as demo/study.py (rng seeded identically, same draw order).""" + rng = random.Random(seed) + scs = [] + ship_cols = [0, 1, 2, 4, 5, 6, 8, 9] + for i in range(n): + ship = rng.choice(ship_cols) + alien = rng.randrange(10) + rows_up = rng.choice([2, 3, 5, 8]) + bomb = None + r = rng.random() + if r < 0.40: + bomb = (max(0, min(9, ship + rng.choice([-1, 0, 0, 1]))), rng.choice([60, 70, 80, 90])) + elif r < 0.65: + bomb = (max(0, min(9, ship + rng.choice([-3, -2, 2, 3]))), rng.choice([30, 45, 55])) + elif r < 0.80: + bomb = (rng.randrange(10), rng.choice([10, 20, 30])) + scs.append(dict(ship=ship, alien=alien, rows=rows_up, bomb=bomb)) + return scs + + +def rep_col(sc): + s = (f"Space Invaders. Your ship is in column {sc['ship']} of 10. " + f"40 aliens remain; the lowest is at column {sc['alien']}, {sc['rows']} rows above you. ") + if sc["bomb"]: + s += (f"One bomb is falling at column {sc['bomb'][0]}, {sc['bomb'][1]} percent from the top" + + (", close to your ship" if sc["bomb"][0] == sc["ship"] and sc["bomb"][1] >= 55 else "") + + ". ") + else: + s += "No bombs in the air. " + return s + "Your gun is ready." + + +def y_danger(sc): + b = sc["bomb"] + return b is not None and b[0] == sc["ship"] and b[1] >= 55 + + +def y_fire(sc): + return sc["alien"] == sc["ship"] + + +def y_bomb_left(sc): + b = sc["bomb"] + return b is not None and b[0] < sc["ship"] + + +def y_alien_left(sc): + return sc["alien"] < sc["ship"] + + +def y_dodge_dir(sc): + b = sc["bomb"] + if b is None or abs(b[0] - sc["ship"]) > 2 or b[1] < 40: + return 0 + return 1 if b[0] < sc["ship"] else -1 if b[0] > sc["ship"] else 0 + + +def ask(url, state, questions): + payload = {"model": "jev-latest", "state": state, "questions": questions} + req = urllib.request.Request(url, data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=60) as r: + return json.loads(r.read()) + + +def main(): + scs = make_scenarios(120) + print(f"n={len(scs)} scenarios; sending the {len(QUESTIONS)}-question batch to both servers") + + cpu_rows, gpu_rows = [], [] + t0 = time.time() + for i, sc in enumerate(scs): + state = rep_col(sc) + gpu = ask(GPU, state, QUESTIONS)["answers"] + gpu_rows.append((sc, gpu)) + time.sleep(0.3) + cpu = ask(CPU, state, QUESTIONS)["answers"] + cpu_rows.append((sc, cpu)) + time.sleep(0.2) + if (i + 1) % 20 == 0: + print(f" {i+1}/{len(scs)} done ({time.time()-t0:.0f}s)") + + # agreement + per_q = {} + noul_dps = {} + choice_agree = [] + for (sc, gpu), (_, cpu) in zip(gpu_rows, cpu_rows): + for qname in QUESTIONS: + g, c = gpu[qname], cpu[qname] + if qname == "dodge/choice": + lab = y_dodge_dir(sc) + if lab != 0: + gg = g["probabilities"]["right"] > 0.5 + cc = c["probabilities"]["right"] > 0.5 + want = lab > 0 + choice_agree.append((gg == want, cc == want, gg == cc)) + else: + gg, cc = g["noul"], c["noul"] + d = abs(gg - cc) + noul_dps.setdefault(qname, []).append(d) + key = qname.split("/")[0] if not qname.startswith("spatial/") else qname + lab = {"danger": y_danger, "fire": y_fire, + "spatial/bombL": y_bomb_left, "spatial/alienL": y_alien_left}[key](sc) + per_q.setdefault(qname, []).append(((gg > 0.5) == lab, (cc > 0.5) == lab)) + + results = {"scenarios": len(scs), "per_question": {}} + print(f"\n{'question':16s} {'agree%':>8s} {'cpu_acc':>8s} {'gpu_acc':>8s} {'mean|dP|':>9s} {'p95|dP|':>8s}") + tot_agree = tot_n = 0 + for qname in QUESTIONS: + if qname == "dodge/choice": + n = len(choice_agree) + agree = sum(1 for _, _, same in choice_agree if same) + cpu_ok = sum(1 for _, ok, _ in choice_agree if ok) + gpu_ok = sum(1 for ok, _, _ in choice_agree if ok) + print(f"{qname:16s} {100*agree/n:7.1f}% {100*cpu_ok/n:7.1f}% {100*gpu_ok/n:7.1f}%") + results["per_question"][qname] = dict(n=n, agree=round(agree/n, 4)) + tot_agree += agree; tot_n += n + continue + rows = per_q[qname] + n = len(rows) + agree = sum(1 for g, c in rows if g == c) + cpu_ok = sum(1 for _, c in rows if c) + gpu_ok = sum(1 for g, _ in rows if g) + dps = noul_dps[qname] + dps_sorted = sorted(dps) + p95 = dps_sorted[int(0.95 * (n - 1))] + print(f"{qname:16s} {100*agree/n:7.1f}% {100*cpu_ok/n:7.1f}% {100*gpu_ok/n:7.1f}% " + f"{statistics.mean(dps):9.4f} {p95:8.4f}") + results["per_question"][qname] = dict(n=n, agree=round(agree/n, 4), cpu_acc=round(cpu_ok/n, 4), + gpu_acc=round(gpu_ok/n, 4), mean_dp=round(statistics.mean(dps), 4), + p95_dp=round(p95, 4), max_dp=round(max(dps), 4)) + tot_agree += agree; tot_n += n + print(f"{'OVERALL':16s} {100*tot_agree/tot_n:7.1f}%") + results["overall_agree"] = round(tot_agree / tot_n, 4) + + # ------------------------------------------------------------------ latency + def pctl(xs, p): + xs = sorted(xs) + return xs[int(p / 100 * (len(xs) - 1))] + + def timed(url, state, questions, runs=30): + xs = [] + for i in range(runs): + t = time.perf_counter() + ask(url, state, questions) + xs.append((time.perf_counter() - t) * 1000) + if url == CPU: + time.sleep(0.5) # ~2 req/s cap against the CPU server + else: + time.sleep(0.05) + return xs + + short_q = {"danger/hit": QUESTIONS["danger/hit"]} + long_state = ("Refund policy section 4: " + + "Items reported missing within thirty days of delivery are refunded to the original payment method. " * 9 + + "The customer says the box arrived empty and this is the second time. ") + long_qs = { + "refund": {"type": "noul", "instructions": "Should this customer get a refund?"}, + "policy": {"type": "noul", "instructions": "Does the report arrive within the policy window?"}, + "repeat": {"type": "noul", "instructions": "Is this the customer's first complaint?"}, + "team": {"type": "choice", "instructions": "Which team should handle this message?", + "criteria": {"billing": "payments, refunds", "shipping": "deliveries, parcels", "tech": "a product that does not work"}}, + "anger": {"type": "score", "instructions": "How angry is the customer?", "criteria": ["calm", "annoyed", "angry", "furious"]}, + } + + lat = {} + for name, state, qs in [("short/1q", rep_col(scs[0]), short_q), ("long/5q", long_state, long_qs)]: + for label, url in [("cpu", CPU), ("gpu", GPU)]: + xs = timed(url, state, qs) + lat[f"{name}"] = lat.get(f"{name}", {}) + lat[f"{name}"][label] = dict(p50_ms=round(pctl(xs, 50), 1), p95_ms=round(pctl(xs, 95), 1), + mean_ms=round(statistics.mean(xs), 1), n=len(xs)) + print(f"{name} {label}: p50={pctl(xs,50):.1f}ms p95={pctl(xs,95):.1f}ms (n={len(xs)})") + results["latency"] = lat + + with open("results.json", "w") as f: + json.dump(results, f, indent=1) + print("\nwrote results.json") + + +if __name__ == "__main__": + main()